From 352513dbc6b1186fffe21e9375a4d75fd7e881bc Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Thu, 28 Sep 2023 17:11:20 +0300 Subject: [PATCH 1/8] :zap: setup --- .../ItemLists/V3/actions/itemList/index.ts | 18 ++++- .../itemList/manageBinaries.operation.ts | 71 +++++++++++++++++++ .../nodes/ItemLists/V3/actions/node.type.ts | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts index 6cb20a66e307c..359f784fb12ba 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts @@ -2,12 +2,21 @@ import type { INodeProperties } from 'n8n-workflow'; import * as concatenateItems from './concatenateItems.operation'; import * as limit from './limit.operation'; +import * as manageBinaries from './manageBinaries.operation'; import * as removeDuplicates from './removeDuplicates.operation'; import * as sort from './sort.operation'; import * as splitOutItems from './splitOutItems.operation'; import * as summarize from './summarize.operation'; -export { concatenateItems, limit, removeDuplicates, sort, splitOutItems, summarize }; +export { + concatenateItems, + limit, + manageBinaries, + removeDuplicates, + sort, + splitOutItems, + summarize, +}; export const description: INodeProperties[] = [ { @@ -33,6 +42,12 @@ export const description: INodeProperties[] = [ description: 'Remove items if there are too many', action: 'Limit', }, + { + name: 'Manage Binaries', + value: 'manageBinaries', + description: 'Manage binary files', + action: 'Manage Binaries', + }, { name: 'Remove Duplicates', value: 'removeDuplicates', @@ -63,6 +78,7 @@ export const description: INodeProperties[] = [ }, ...concatenateItems.description, ...limit.description, + ...manageBinaries.description, ...removeDuplicates.description, ...sort.description, ...splitOutItems.description, diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts new file mode 100644 index 0000000000000..2eab67da2550b --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts @@ -0,0 +1,71 @@ +import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Perform ...', + name: 'perform', + type: 'options', + options: [ + { + name: 'Merge Binaries Into a Single Item', + value: 'merge', + }, + { + name: 'Split Each Binary Into a Separate Item', + value: 'split', + }, + ], + default: 'merge', + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['manageBinaries'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const returnData = [] as INodeExecutionData[]; + const perform = this.getNodeParameter('perform', 0) as string; + + if (perform === 'merge') { + const newItem = { + json: {}, + } as INodeExecutionData; + + for (const item of items) { + if (item.binary === undefined) continue; + + for (const [index, key] of Object.keys(item.binary).entries()) { + if (!newItem.binary) newItem.binary = {}; + + newItem.binary[`${key}${index}`] = item.binary[key]; + } + } + + returnData.push(newItem); + } else { + for (const item of items) { + if (item.binary === undefined) continue; + + for (const key of Object.keys(item.binary)) { + returnData.push({ + json: item.json, + binary: { + [key]: item.binary[key], + }, + } as INodeExecutionData); + } + } + } + + return returnData; +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts index 86750f30eac1b..0baf55a430697 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts @@ -4,6 +4,7 @@ type NodeMap = { itemList: | 'concatenateItems' | 'limit' + | 'manageBinaries' | 'removeDuplicates' | 'sort' | 'splitOutItems' From adef794842d2bd9e626513c73f52c414e2890b5c Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Fri, 29 Sep 2023 10:14:15 +0300 Subject: [PATCH 2/8] :zap: options --- .../itemList/manageBinaries.operation.ts | 132 +++++++++++++++++- 1 file changed, 126 insertions(+), 6 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts index 2eab67da2550b..5eb38336eb37b 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts @@ -1,4 +1,11 @@ -import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import type { + IDataObject, + IBinaryData, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { jsonParse } from 'n8n-workflow'; import { updateDisplayOptions } from '@utils/utilities'; const properties: INodeProperties[] = [ @@ -18,6 +25,64 @@ const properties: INodeProperties[] = [ ], default: 'merge', }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + options: [ + { + displayName: 'Include JSON Data', + name: 'includeJson', + type: 'boolean', + default: false, + description: 'Whether to include the JSON data in the new items', + displayOptions: { + show: { + '/perform': ['split'], + }, + }, + }, + { + displayName: 'JSON Data', + name: 'json', + type: 'string', + default: '{\n "my_field_1": "value",\n "my_field_2": 1\n}', + typeOptions: { + editor: 'json', + editorLanguage: 'json', + rows: 4, + }, + displayOptions: { + show: { + '/perform': ['merge'], + }, + }, + validateType: 'object', + }, + { + displayName: 'Keep Only Unique Binaries', + name: 'keepOnlyUnique', + type: 'boolean', + default: false, + description: + 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', + }, + { + displayName: 'Rename Binary Key', + name: 'renameBinaryKey', + type: 'string', + default: '', + description: 'If set, the binary key will be renamed to this value', + displayOptions: { + show: { + '/perform': ['split'], + }, + }, + }, + ], + }, ]; const displayOptions = { @@ -29,38 +94,93 @@ const displayOptions = { export const description = updateDisplayOptions(displayOptions, properties); +type PartialBinaryData = Omit; +const isBinaryUniqueSetup = () => { + const binaries: PartialBinaryData[] = []; + return (binary: IBinaryData) => { + for (const existingBinary of binaries) { + if ( + existingBinary.mimeType === binary.mimeType && + existingBinary.fileType === binary.fileType && + existingBinary.fileSize === binary.fileSize && + existingBinary.fileExtension === binary.fileExtension + ) { + return false; + } + } + + binaries.push({ + mimeType: binary.mimeType, + fileType: binary.fileType, + fileSize: binary.fileSize, + fileExtension: binary.fileExtension, + }); + + return true; + }; +}; + export async function execute( this: IExecuteFunctions, items: INodeExecutionData[], ): Promise { const returnData = [] as INodeExecutionData[]; const perform = this.getNodeParameter('perform', 0) as string; + const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; + const isBinaryUnique = keepOnlyUnique ? isBinaryUniqueSetup() : undefined; if (perform === 'merge') { + let jsonData = this.getNodeParameter('options.json', 0, {}) as IDataObject; + if (jsonData && typeof jsonData === 'string') { + jsonData = jsonParse(jsonData); + } + const newItem = { - json: {}, + json: jsonData ? jsonData : {}, } as INodeExecutionData; for (const item of items) { if (item.binary === undefined) continue; - for (const [index, key] of Object.keys(item.binary).entries()) { + for (const key of Object.keys(item.binary)) { if (!newItem.binary) newItem.binary = {}; + let binaryKey = key; + const binary = item.binary[key]; + + if (isBinaryUnique && !isBinaryUnique(binary)) { + continue; + } + + // If the binary key already exists add a suffix to it + let i = 1; + while (newItem.binary[binaryKey] !== undefined) { + binaryKey = `${key}_${i}`; + i++; + } - newItem.binary[`${key}${index}`] = item.binary[key]; + newItem.binary[binaryKey] = binary; } } returnData.push(newItem); } else { + const includeJson = this.getNodeParameter('options.includeJson', 0, false) as boolean; + const renameBinaryKey = this.getNodeParameter('options.renameBinaryKey', 0, '') as string; + for (const item of items) { if (item.binary === undefined) continue; for (const key of Object.keys(item.binary)) { + const binary = item.binary[key]; + + if (isBinaryUnique && !isBinaryUnique(binary)) { + continue; + } + returnData.push({ - json: item.json, + json: includeJson ? item.json : {}, binary: { - [key]: item.binary[key], + [renameBinaryKey || key]: binary, }, } as INodeExecutionData); } From 00d1c39339f389ba30037d82e275f34367497fd9 Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Mon, 2 Oct 2023 11:47:23 +0300 Subject: [PATCH 3/8] :zap: include binaries option to aggregate operation --- .../itemList/concatenateItems.operation.ts | 69 +++++++++++++++--- .../nodes/ItemLists/V3/helpers/utils.ts | 72 +++++++++++++++++-- 2 files changed, 124 insertions(+), 17 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts index c70947df59073..9af543ece2192 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts @@ -13,7 +13,7 @@ import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; import set from 'lodash/set'; -import { prepareFieldsArray } from '../../helpers/utils'; +import { addBinariesToItem, prepareFieldsArray } from '../../helpers/utils'; import { disableDotNotationBoolean } from '../common.descriptions'; const properties: INodeProperties[] = [ @@ -159,13 +159,15 @@ const properties: INodeProperties[] = [ type: 'collection', placeholder: 'Add Field', default: {}, - displayOptions: { - hide: { - aggregate: ['aggregateAllItemData'], - }, - }, options: [ - disableDotNotationBoolean, + { + ...disableDotNotationBoolean, + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, { displayName: 'Merge Lists', name: 'mergeLists', @@ -173,6 +175,31 @@ const properties: INodeProperties[] = [ default: false, description: 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Include Binaries', + name: 'includeBinaries', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new item', + }, + { + displayName: 'Keep Only Unique Binaries', + name: 'keepOnlyUnique', + type: 'boolean', + default: false, + description: + 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', + displayOptions: { + show: { + includeBinaries: [true], + }, + }, }, { displayName: 'Keep Missing And Null Values', @@ -181,6 +208,11 @@ const properties: INodeProperties[] = [ default: false, description: 'Whether to add a null entry to the aggregated list when there is a missing or null value', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, }, ], }, @@ -199,7 +231,7 @@ export async function execute( this: IExecuteFunctions, items: INodeExecutionData[], ): Promise { - const returnData: INodeExecutionData[] = []; + let returnData: INodeExecutionData = { json: {}, pairedItem: [] }; const aggregate = this.getNodeParameter('aggregate', 0, '') as string; @@ -305,7 +337,7 @@ export async function execute( } } - returnData.push(newItem); + returnData = newItem; } else { let newItems: IDataObject[] = items.map((item) => item.json); let pairedItem: IPairedItemData[] = []; @@ -353,8 +385,23 @@ export async function execute( } const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem }; - returnData.push(output); + + returnData = output; + } + + const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean; + + if (includeBinaries) { + const pairedItems = (returnData.pairedItem || []) as IPairedItemData[]; + + const aggregatedItems = pairedItems.map((item) => { + return items[item.item]; + }); + + const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; + + addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique); } - return returnData; + return [returnData]; } diff --git a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts index 94ba6a2db8695..8f2337ec4d065 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts @@ -1,11 +1,12 @@ import { NodeVM } from '@n8n/vm2'; -import { - NodeOperationError, - type IDataObject, - type IExecuteFunctions, - type INode, - type INodeExecutionData, +import type { + IDataObject, + IExecuteFunctions, + IBinaryData, + INode, + INodeExecutionData, } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; import get from 'lodash/get'; import isEqual from 'lodash/isEqual'; @@ -87,3 +88,62 @@ export function sortByCode( return vm.run(`module.exports = items.sort((a, b) => { ${code} })`); } + +type PartialBinaryData = Omit; +const isBinaryUniqueSetup = () => { + const binaries: PartialBinaryData[] = []; + return (binary: IBinaryData) => { + for (const existingBinary of binaries) { + if ( + existingBinary.mimeType === binary.mimeType && + existingBinary.fileType === binary.fileType && + existingBinary.fileSize === binary.fileSize && + existingBinary.fileExtension === binary.fileExtension + ) { + return false; + } + } + + binaries.push({ + mimeType: binary.mimeType, + fileType: binary.fileType, + fileSize: binary.fileSize, + fileExtension: binary.fileExtension, + }); + + return true; + }; +}; + +export function addBinariesToItem( + newItem: INodeExecutionData, + items: INodeExecutionData[], + uniqueOnly?: boolean, +) { + const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined; + + for (const item of items) { + if (item.binary === undefined) continue; + + for (const key of Object.keys(item.binary)) { + if (!newItem.binary) newItem.binary = {}; + let binaryKey = key; + const binary = item.binary[key]; + + if (isBinaryUnique && !isBinaryUnique(binary)) { + continue; + } + + // If the binary key already exists add a suffix to it + let i = 1; + while (newItem.binary[binaryKey] !== undefined) { + binaryKey = `${key}_${i}`; + i++; + } + + newItem.binary[binaryKey] = binary; + } + } + + return newItem; +} From 4136d2699573e43e8f22e0207fd6fea9cca9492e Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Mon, 2 Oct 2023 12:30:44 +0300 Subject: [PATCH 4/8] :zap: include binary option for splited items --- .../V3/actions/itemList/splitOutItems.operation.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts index 68bd5dbfbee71..b7bfed2c6cc78 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -74,6 +74,13 @@ const properties: INodeProperties[] = [ default: '', description: 'The field in the output under which to put the split field contents', }, + { + displayName: 'Include Binary', + name: 'includeBinary', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new items', + }, ], }, ]; @@ -103,6 +110,8 @@ export async function execute( false, ) as boolean; + const includeBinary = this.getNodeParameter('options.includeBinary', i, false) as boolean; + const destinationFields = ( this.getNodeParameter('options.destinationFieldName', i, '') as string ) @@ -211,6 +220,7 @@ export async function execute( returnData.push({ json: newItem, + binary: includeBinary ? items[i].binary : undefined, pairedItem: { item: i, }, From eca9e98f5b534479a9bb33e8a4affdce626eb65d Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Fri, 6 Oct 2023 16:48:19 +0300 Subject: [PATCH 5/8] :zap: adding support to split by binaries to splitItems, WIP --- .../itemList/splitOutItems.operation.ts | 90 +++++++++++-------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts index b7bfed2c6cc78..9c7727ed873ce 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -1,4 +1,5 @@ import type { + IBinaryData, IDataObject, IExecuteFunctions, INodeExecutionData, @@ -103,14 +104,15 @@ export async function execute( for (let i = 0; i < items.length; i++) { const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string) .split(',') - .map((field) => field.trim()); + .map((field) => field.trim().replace(/^\$json\./, '')); + const disableDotNotation = this.getNodeParameter( 'options.disableDotNotation', 0, false, ) as boolean; - const includeBinary = this.getNodeParameter('options.includeBinary', i, false) as boolean; + // const includeBinary = this.getNodeParameter('options.includeBinary', i, false) as boolean; const destinationFields = ( this.getNodeParameter('options.destinationFieldName', i, '') as string @@ -134,54 +136,74 @@ export async function execute( const multiSplit = fieldsToSplitOut.length > 1; const item = { ...items[i].json }; - const splited: IDataObject[] = []; + const splited: INodeExecutionData[] = []; for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) { const destinationFieldName = destinationFields[entryIndex] || ''; - let arrayToSplit; - if (!disableDotNotation) { - arrayToSplit = get(item, fieldToSplitOut); + let entityToSplit: IDataObject[] = []; + + if (fieldToSplitOut === '$binary') { + entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({ + [key]: value, + })); + } else if (fieldToSplitOut === '$json') { + entityToSplit = Object.values(item) as IDataObject[]; } else { - arrayToSplit = item[fieldToSplitOut]; - } + if (!disableDotNotation) { + entityToSplit = get(item, fieldToSplitOut) as IDataObject[]; + } else { + entityToSplit = item[fieldToSplitOut] as IDataObject[]; + } - if (arrayToSplit === undefined) { - arrayToSplit = []; - } + if (entityToSplit === undefined) { + entityToSplit = []; + } - if (typeof arrayToSplit !== 'object' || arrayToSplit === null) { - arrayToSplit = [arrayToSplit]; - } + if (typeof entityToSplit !== 'object' || entityToSplit === null) { + entityToSplit = [entityToSplit]; + } - if (!Array.isArray(arrayToSplit)) { - arrayToSplit = Object.values(arrayToSplit); + if (!Array.isArray(entityToSplit)) { + entityToSplit = Object.values(entityToSplit); + } } - for (const [elementIndex, element] of arrayToSplit.entries()) { + for (const [elementIndex, element] of entityToSplit.entries()) { if (splited[elementIndex] === undefined) { - splited[elementIndex] = {}; + splited[elementIndex] = { json: {}, pairedItem: { item: i } }; } const fieldName = destinationFieldName || fieldToSplitOut; + let json = element; - if (typeof element === 'object' && element !== null && include === 'noOtherFields') { + if (fieldToSplitOut === '$binary') { + if (splited[elementIndex].binary === undefined) { + splited[elementIndex].binary = {}; + } + splited[elementIndex].binary![Object.keys(element)[0]] = Object.values( + element, + )[0] as IBinaryData; + + json = {}; + } + + if (typeof json === 'object' && json !== null && include === 'noOtherFields') { if (destinationFieldName === '' && !multiSplit) { - splited[elementIndex] = { ...splited[elementIndex], ...element }; + splited[elementIndex] = { + json: { ...splited[elementIndex].json, ...json }, + pairedItem: { item: i }, + }; } else { - splited[elementIndex][fieldName] = element; + splited[elementIndex].json[fieldName] = json; } } else { - splited[elementIndex][fieldName] = element; + splited[elementIndex][fieldName] = json; } } } for (const splitEntry of splited) { - let newItem: IDataObject = {}; - - if (include === 'noOtherFields') { - newItem = splitEntry; - } + let newItem: INodeExecutionData = splitEntry; if (include === 'allOtherFields') { const itemCopy = deepCopy(item); @@ -192,7 +214,7 @@ export async function execute( delete itemCopy[fieldToSplitOut]; } } - newItem = { ...itemCopy, ...splitEntry }; + newItem.json = { ...itemCopy, ...splitEntry.json }; } if (include === 'selectedOtherFields') { @@ -209,22 +231,16 @@ export async function execute( for (const field of fieldsToInclude) { if (!disableDotNotation) { - splitEntry[field] = get(item, field); + splitEntry.json[field] = get(item, field); } else { - splitEntry[field] = item[field]; + splitEntry.json[field] = item[field]; } } newItem = splitEntry; } - returnData.push({ - json: newItem, - binary: includeBinary ? items[i].binary : undefined, - pairedItem: { - item: i, - }, - }); + returnData.push(newItem); } } From 7fd5b0e63bdf41fa0f497e45f76aa43969777dbf Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Mon, 9 Oct 2023 10:56:55 +0300 Subject: [PATCH 6/8] :zap: split by support --- .../ItemLists/V3/actions/itemList/index.ts | 18 +- .../itemList/manageBinaries.operation.ts | 191 ------------------ .../itemList/splitOutItems.operation.ts | 35 ++-- .../nodes/ItemLists/V3/actions/node.type.ts | 1 - 4 files changed, 19 insertions(+), 226 deletions(-) delete mode 100644 packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts index 359f784fb12ba..6cb20a66e307c 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts @@ -2,21 +2,12 @@ import type { INodeProperties } from 'n8n-workflow'; import * as concatenateItems from './concatenateItems.operation'; import * as limit from './limit.operation'; -import * as manageBinaries from './manageBinaries.operation'; import * as removeDuplicates from './removeDuplicates.operation'; import * as sort from './sort.operation'; import * as splitOutItems from './splitOutItems.operation'; import * as summarize from './summarize.operation'; -export { - concatenateItems, - limit, - manageBinaries, - removeDuplicates, - sort, - splitOutItems, - summarize, -}; +export { concatenateItems, limit, removeDuplicates, sort, splitOutItems, summarize }; export const description: INodeProperties[] = [ { @@ -42,12 +33,6 @@ export const description: INodeProperties[] = [ description: 'Remove items if there are too many', action: 'Limit', }, - { - name: 'Manage Binaries', - value: 'manageBinaries', - description: 'Manage binary files', - action: 'Manage Binaries', - }, { name: 'Remove Duplicates', value: 'removeDuplicates', @@ -78,7 +63,6 @@ export const description: INodeProperties[] = [ }, ...concatenateItems.description, ...limit.description, - ...manageBinaries.description, ...removeDuplicates.description, ...sort.description, ...splitOutItems.description, diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts deleted file mode 100644 index 5eb38336eb37b..0000000000000 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/manageBinaries.operation.ts +++ /dev/null @@ -1,191 +0,0 @@ -import type { - IDataObject, - IBinaryData, - IExecuteFunctions, - INodeExecutionData, - INodeProperties, -} from 'n8n-workflow'; -import { jsonParse } from 'n8n-workflow'; -import { updateDisplayOptions } from '@utils/utilities'; - -const properties: INodeProperties[] = [ - { - displayName: 'Perform ...', - name: 'perform', - type: 'options', - options: [ - { - name: 'Merge Binaries Into a Single Item', - value: 'merge', - }, - { - name: 'Split Each Binary Into a Separate Item', - value: 'split', - }, - ], - default: 'merge', - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - options: [ - { - displayName: 'Include JSON Data', - name: 'includeJson', - type: 'boolean', - default: false, - description: 'Whether to include the JSON data in the new items', - displayOptions: { - show: { - '/perform': ['split'], - }, - }, - }, - { - displayName: 'JSON Data', - name: 'json', - type: 'string', - default: '{\n "my_field_1": "value",\n "my_field_2": 1\n}', - typeOptions: { - editor: 'json', - editorLanguage: 'json', - rows: 4, - }, - displayOptions: { - show: { - '/perform': ['merge'], - }, - }, - validateType: 'object', - }, - { - displayName: 'Keep Only Unique Binaries', - name: 'keepOnlyUnique', - type: 'boolean', - default: false, - description: - 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', - }, - { - displayName: 'Rename Binary Key', - name: 'renameBinaryKey', - type: 'string', - default: '', - description: 'If set, the binary key will be renamed to this value', - displayOptions: { - show: { - '/perform': ['split'], - }, - }, - }, - ], - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['manageBinaries'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -type PartialBinaryData = Omit; -const isBinaryUniqueSetup = () => { - const binaries: PartialBinaryData[] = []; - return (binary: IBinaryData) => { - for (const existingBinary of binaries) { - if ( - existingBinary.mimeType === binary.mimeType && - existingBinary.fileType === binary.fileType && - existingBinary.fileSize === binary.fileSize && - existingBinary.fileExtension === binary.fileExtension - ) { - return false; - } - } - - binaries.push({ - mimeType: binary.mimeType, - fileType: binary.fileType, - fileSize: binary.fileSize, - fileExtension: binary.fileExtension, - }); - - return true; - }; -}; - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - const returnData = [] as INodeExecutionData[]; - const perform = this.getNodeParameter('perform', 0) as string; - const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; - const isBinaryUnique = keepOnlyUnique ? isBinaryUniqueSetup() : undefined; - - if (perform === 'merge') { - let jsonData = this.getNodeParameter('options.json', 0, {}) as IDataObject; - if (jsonData && typeof jsonData === 'string') { - jsonData = jsonParse(jsonData); - } - - const newItem = { - json: jsonData ? jsonData : {}, - } as INodeExecutionData; - - for (const item of items) { - if (item.binary === undefined) continue; - - for (const key of Object.keys(item.binary)) { - if (!newItem.binary) newItem.binary = {}; - let binaryKey = key; - const binary = item.binary[key]; - - if (isBinaryUnique && !isBinaryUnique(binary)) { - continue; - } - - // If the binary key already exists add a suffix to it - let i = 1; - while (newItem.binary[binaryKey] !== undefined) { - binaryKey = `${key}_${i}`; - i++; - } - - newItem.binary[binaryKey] = binary; - } - } - - returnData.push(newItem); - } else { - const includeJson = this.getNodeParameter('options.includeJson', 0, false) as boolean; - const renameBinaryKey = this.getNodeParameter('options.renameBinaryKey', 0, '') as string; - - for (const item of items) { - if (item.binary === undefined) continue; - - for (const key of Object.keys(item.binary)) { - const binary = item.binary[key]; - - if (isBinaryUnique && !isBinaryUnique(binary)) { - continue; - } - - returnData.push({ - json: includeJson ? item.json : {}, - binary: { - [renameBinaryKey || key]: binary, - }, - } as INodeExecutionData); - } - } - } - - return returnData; -} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts index 9c7727ed873ce..0abae82b20e16 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -106,17 +106,11 @@ export async function execute( .split(',') .map((field) => field.trim().replace(/^\$json\./, '')); - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; + const options = this.getNodeParameter('options', i, {}); - // const includeBinary = this.getNodeParameter('options.includeBinary', i, false) as boolean; + const disableDotNotation = options.disableDotNotation as boolean; - const destinationFields = ( - this.getNodeParameter('options.destinationFieldName', i, '') as string - ) + const destinationFields = ((options.destinationFieldName as string) || '') .split(',') .filter((field) => field.trim() !== '') .map((field) => field.trim()); @@ -146,8 +140,6 @@ export async function execute( entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({ [key]: value, })); - } else if (fieldToSplitOut === '$json') { - entityToSplit = Object.values(item) as IDataObject[]; } else { if (!disableDotNotation) { entityToSplit = get(item, fieldToSplitOut) as IDataObject[]; @@ -174,7 +166,6 @@ export async function execute( } const fieldName = destinationFieldName || fieldToSplitOut; - let json = element; if (fieldToSplitOut === '$binary') { if (splited[elementIndex].binary === undefined) { @@ -184,20 +175,20 @@ export async function execute( element, )[0] as IBinaryData; - json = {}; + continue; } - if (typeof json === 'object' && json !== null && include === 'noOtherFields') { + if (typeof element === 'object' && element !== null && include === 'noOtherFields') { if (destinationFieldName === '' && !multiSplit) { splited[elementIndex] = { - json: { ...splited[elementIndex].json, ...json }, + json: { ...splited[elementIndex].json, ...element }, pairedItem: { item: i }, }; } else { - splited[elementIndex].json[fieldName] = json; + splited[elementIndex].json[fieldName] = element; } } else { - splited[elementIndex][fieldName] = json; + splited[elementIndex].json[fieldName] = element; } } } @@ -240,6 +231,16 @@ export async function execute( newItem = splitEntry; } + const includeBinary = options.includeBinary as boolean; + + if (includeBinary) { + if (items[i].binary && !newItem.binary) { + newItem.binary = items[i].binary; + } + } else if (includeBinary !== undefined) { + delete newItem.binary; + } + returnData.push(newItem); } } diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts index 0baf55a430697..86750f30eac1b 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts @@ -4,7 +4,6 @@ type NodeMap = { itemList: | 'concatenateItems' | 'limit' - | 'manageBinaries' | 'removeDuplicates' | 'sort' | 'splitOutItems' From 54e749dc0e2e680ad09eb36f3dbe8c52976fb5e4 Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Mon, 9 Oct 2023 11:05:25 +0300 Subject: [PATCH 7/8] :zap: added placeholder, updated description --- .../ItemLists/V3/actions/itemList/splitOutItems.operation.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts index 0abae82b20e16..f40e207310403 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -21,7 +21,9 @@ const properties: INodeProperties[] = [ type: 'string', default: '', required: true, - description: 'The name of the input fields to break out into separate items', + placeholder: 'Drag fields from the left or type their names', + description: + 'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.', requiresDataPath: 'multiple', }, { From a043e0de69f2fa4f55acb8ea7bdd85ade9f9c597 Mon Sep 17 00:00:00 2001 From: Marcus Date: Tue, 10 Oct 2023 16:24:22 +0200 Subject: [PATCH 8/8] split out items - do not delete binary for new items to support $binary fields --- .../ItemLists/V3/actions/itemList/splitOutItems.operation.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts index f40e207310403..35b162ef870e1 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -239,8 +239,6 @@ export async function execute( if (items[i].binary && !newItem.binary) { newItem.binary = items[i].binary; } - } else if (includeBinary !== undefined) { - delete newItem.binary; } returnData.push(newItem);