diff --git a/CHANGELOG.md b/CHANGELOG.md index dc902e7..a653e8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,26 @@ # Changelog -## v1.0.0 +## v1.1.0 -#### :rocket: Features +#### :rocket: Added Features + +- Added options to convert transform string values into map using a pluginType named `transformPropertyToMap` + +#### :fire: Removed +- Removed files `mergeUsers.js` and `mergeGroups.js` which was being unused -- Merge all the `CSV` files and convert into single `JSON` file. -- Merge all the `CSV` files, convert it to `JSON` object, assign it to a `JSON field` an then store it in a `JSON` file. -- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field and then store it to a `JSON` file. -- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field, assign the object to a `JSON` field and then store it to a `JSON` file. ## v1.0.1 #### :bug: Bug Fixes - Removed the code which was trying to read a file which didn't existed, a junk which creeped during the development cycle. + +## v1.0.0 + +#### :rocket: Features + +- Merge all the `CSV` files and convert into single `JSON` file. +- Merge all the `CSV` files, convert it to `JSON` object, assign it to a `JSON field` an then store it in a `JSON` file. +- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field and then store it to a `JSON` file. +- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field, assign the object to a `JSON` field and then store it to a `JSON` file. diff --git a/README.md b/README.md index 1a2a995..79889c6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Merges all the csv files and converts it to json. - Merge all the `CSV` files, convert it to `JSON` object, assign it to a `JSON field` an then store it in a `JSON` file. - Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field and then store it to a `JSON` file. - Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field, assign the object to a `JSON` field and then store it to a `JSON` file. - +- Merge all the `CSV` files, convert into single `JSON` file by converting string values into map by providing `keyValueSplitter`, `propertySplitter`, `keyIndex`, `valueIndex`, `field` and `pluginType` as `transfromPropertyToMap` ## Installation @@ -53,3 +53,29 @@ yarn add merge-convert-csv-to-json ``` merge-convert --pattern "/users/*.users.csv" --keyBy email --assignTo users --outputTo /users/merged.users.json ``` +- Merge all the `CSV` files, convert into single `JSON` file by converting string values into map by providing `keyValueSplitter`, `propertySplitter`, `keyIndex`, `valueIndex`, `field` and `pluginType` as `transfromPropertyToMap` + ```json + # transformer-config.json + [ + { + pluginType: "transformPropertyToMap", + keyValueSplitter: ":", + propertySplitter: ",", + keyIndex: "0", + valueIndex: "1", + field: "groups_access" + }, + { + pluginType: "transformPropertyToMap", + keyValueSplitter: ":", + propertySplitter: ",", + keyIndex: "0", + valueIndex: "1", + field: "project_access" + } + ] + ``` + + ``` + merge-convert --pattern "/users/*.users.csv" --outputTo /users/merged.users.json --transformerConfig transformer-config.json + ``` diff --git a/__tests__/bin/merge-convert.js b/__tests__/bin/merge-convert.js index aad3825..e1b4c9d 100644 --- a/__tests__/bin/merge-convert.js +++ b/__tests__/bin/merge-convert.js @@ -5,15 +5,21 @@ const fs = require('fs'); beforeEach(() => { const sampleA = 'username,email,organization,password\ntommy.jones,tommy.jones@northwindtraders.com,NorthwindTraders,password@123\n'; const sampleB = 'username,email,organization,password\nanne.frank,anne.frank@northwindtraders.com,NorthwindTraders,password@123\n'; + const sampleC = 'username,email,organization,password,groups_access,project_access\nanne.frank,anne.frank@northwindtraders.com,NorthwindTraders,password@123,"northwind-wave-1:guest,northwind-auditors:guest","northwind-wave-1:guest,northwind-auditors:guest"'; + const transformerConfig = '[{"pluginType":"transformPropertyToMap","keyValueSplitter":":","propertySplitter":",","keyIndex":"0","valueIndex":"1","field":"groups_access"},{"pluginType":"transformPropertyToMap","keyValueSplitter":":","propertySplitter":",","keyIndex":"0","valueIndex":"1","field":"project_access"}]'; fs.writeFileSync('/tmp/a.users.csv', sampleA); fs.writeFileSync('/tmp/b.users.csv', sampleB); + fs.writeFileSync('/tmp/c.groups.csv', sampleC); + fs.writeFileSync('/tmp/transformerConfig.json', transformerConfig); }); afterEach(() => { fs.unlinkSync('/tmp/a.users.csv'); fs.unlinkSync('/tmp/b.users.csv'); fs.unlinkSync('/tmp/merged.users.json'); + fs.unlinkSync('/tmp/c.groups.csv'); + fs.unlinkSync('/tmp/transformerConfig.json'); }); @@ -124,3 +130,33 @@ test('Should Merge all the `CSV` files, convert it to `JSON` with every object * done(); }); }); + +test('Should merge all the CSV files and convert into single JSON file with Transformers', (done) => { + const pathToMergeConvert = path.join(__dirname, '../../bin/merge-convert.js'); + const cmd = `${pathToMergeConvert} --pattern "/tmp/c.groups.csv" --outputTo /tmp/merged.users.json --transformerConfig /tmp/transformerConfig.json`; + const expected = [{ + username: 'anne.frank', + email: 'anne.frank@northwindtraders.com', + organization: 'NorthwindTraders', + password: 'password@123', + groups_access: { + 'northwind-wave-1': 'guest', + 'northwind-auditors': 'guest', + }, + project_access: { + 'northwind-wave-1': 'guest', + 'northwind-auditors': 'guest', + }, + }]; + childProcess.exec(cmd, (err, stdout, stderr) => { + if (!err) { + const mergedJSONAsString = fs.readFileSync('/tmp/merged.users.json', 'utf8'); + const mergedJSON = JSON.parse(mergedJSONAsString); + expect(mergedJSON).toMatchObject(expected); + } else { + console.log(stdout); + console.log(stderr); + } + done(); + }); +}); diff --git a/__tests__/lib/concatFiles.js b/__tests__/lib/concatFiles.js index 2298418..c642803 100644 --- a/__tests__/lib/concatFiles.js +++ b/__tests__/lib/concatFiles.js @@ -1,6 +1,7 @@ const mock = require('mock-fs'); const { concatFiles } = require('../../lib/concatFiles'); +const { csvToJson } = require('../../lib/csvToJson'); beforeAll(() => mock({ '/tmp/a.csv': 'username,email,organization,password\ntommy.jones,tommy.jones@northwindtraders.com,NorthwindTraders,password@123\n', @@ -26,14 +27,15 @@ test('Given set of csv files it should return the merged json', (done) => { expect(values).toEqual(expect.arrayContaining(expected)); done(err); }; - concatFiles(['/tmp/a.csv', '/tmp/b.csv'])(testCb); + concatFiles(['/tmp/a.csv', '/tmp/b.csv'], csvToJson)(testCb); }); test('Given a non existing path it should throw error', (done) => { const testCb = (err) => { if (err) done(); }; - concatFiles(['/tmp/d.csv'])(testCb); + + concatFiles(['/tmp/d.csv'], csvToJson)(testCb); }); afterAll(async () => { diff --git a/__tests__/lib/mergeGroups.js b/__tests__/lib/mergeGroups.js deleted file mode 100644 index b1df948..0000000 --- a/__tests__/lib/mergeGroups.js +++ /dev/null @@ -1,38 +0,0 @@ -const fs = require('fs'); -const mock = require('mock-fs'); -const { mergeGroups } = require('../../lib/mergeGroups'); - -beforeAll(() => mock({ - '/tmp/a.group.csv': 'group_name,group_description\ncgi-wave-1,A Group for entire CGI Wave 1\n', - '/tmp/b.group.csv': 'group_name,group_description\nNorthwindTraders-mentors,Group Created for NorthwindTraders mentors\n', - '/tmp/groups.json': '', -})); - -test('Test Merge Groups', (done) => { - const groupFiles = [ - '/tmp/a.group.csv', - '/tmp/b.group.csv', - ]; - - const testCb = (err) => { - const expected = { - 'cgi-wave-1': { - group_name: 'cgi-wave-1', - group_description: 'A Group for entire CGI Wave 1', - }, - 'NorthwindTraders-mentors': { - group_name: 'NorthwindTraders-mentors', - group_description: 'Group Created for NorthwindTraders mentors', - }, - }; - const actual = JSON.parse(fs.readFileSync('/tmp/groups.json', 'utf8')); - expect(actual).toMatchObject(expected); - done(err); - }; - - mergeGroups(groupFiles, '/tmp/groups.json', testCb); -}); - -afterAll(() => { - mock.restore(); -}); diff --git a/__tests__/lib/mergeUsers.js b/__tests__/lib/mergeUsers.js deleted file mode 100644 index 4ffe2f5..0000000 --- a/__tests__/lib/mergeUsers.js +++ /dev/null @@ -1,42 +0,0 @@ -const fs = require('fs'); -const mock = require('mock-fs'); -const { mergeUsers } = require('../../lib/mergeUsers'); - -beforeAll(() => mock({ - '/tmp/a.users.csv': 'username,email,organization,password\ntommy.jones,tommy.jones@northwindtraders.com,NorthwindTraders,password@123\n', - '/tmp/b.users.csv': 'username,email,organization,password\nanne.frank,anne.frank@northwindtraders.com,NorthwindTraders,password@123\n', - '/tmp/users.json': '', -})); - -test('Test Merge Groups', (done) => { - const userFiles = [ - '/tmp/a.users.csv', - '/tmp/b.users.csv', - ]; - - const testCb = (err) => { - const expected = { - 'tommy.jones@northwindtraders.com': { - username: 'tommy.jones', - email: 'tommy.jones@northwindtraders.com', - organization: 'NorthwindTraders', - password: 'password@123', - }, - 'anne.frank@northwindtraders.com': { - username: 'anne.frank', - email: 'anne.frank@northwindtraders.com', - organization: 'NorthwindTraders', - password: 'password@123', - }, - }; - const actual = JSON.parse(fs.readFileSync('/tmp/users.json', 'utf8')); - expect(actual).toMatchObject(expected); - done(err); - }; - - mergeUsers(userFiles, '/tmp/users.json', testCb); -}); - -afterAll(() => { - mock.restore(); -}); diff --git a/__tests__/lib/transformPropertyToMap.js b/__tests__/lib/transformPropertyToMap.js new file mode 100644 index 0000000..5795e15 --- /dev/null +++ b/__tests__/lib/transformPropertyToMap.js @@ -0,0 +1,39 @@ + +const { transformPropertyToMap } = require('../../lib/transformPropertyToMap.js'); + +test('Should be able to convert value of type string to map', (done) => { + const transformerConfig = { + pluginType: 'transformPropertyToMap', + keyValueSplitter: ':', + propertySplitter: ',', + keyIndex: '0', + valueIndex: '1', + field: 'groups_access', + }; + + const data = [ + { + username: 'john.doe', + email: 'john.doe@northwind.in', + password: 'password@123', + groups_access: 'northwind-group:guest,northwind-auditors:guest', + }, + ]; + + const expected = [ + { + username: 'john.doe', + email: 'john.doe@northwind.in', + password: 'password@123', + groups_access: { + 'northwind-group': 'guest', + 'northwind-auditors': 'guest', + }, + }, + ]; + + transformPropertyToMap(transformerConfig, data, (err, transformedData) => { + expect(transformedData).toMatchObject(expected); + done(); + }); +}); diff --git a/bin/merge-convert.js b/bin/merge-convert.js index 57094d5..425eea0 100755 --- a/bin/merge-convert.js +++ b/bin/merge-convert.js @@ -10,6 +10,7 @@ const optionDefinitions = [ { name: 'keyBy', type: String }, { name: 'assignTo', type: String }, { name: 'outputTo', type: String }, + { name: 'transformerConfig', type: String }, ]; try { diff --git a/lib/cli-usage.js b/lib/cli-usage.js index 9507b54..e411de8 100644 --- a/lib/cli-usage.js +++ b/lib/cli-usage.js @@ -28,6 +28,11 @@ const sections = [ typeLabel: '{underline filePath}', description: 'The filepath to output the result to.', }, + { + name: 'transformerConfig', + typeLabel: '{underline propertyName}', + description: 'The name of the file containing the config for transformation', + }, ], }, ]; diff --git a/lib/concatFiles.js b/lib/concatFiles.js index 31ee75e..25e4b3d 100644 --- a/lib/concatFiles.js +++ b/lib/concatFiles.js @@ -1,17 +1,16 @@ const async = require('async'); const { getLogger } = require('./logger'); -const { csvToJson } = require('./csvToJson'); const log = getLogger('concatFiles'); const concatFilesCb = (callback) => (err, values) => callback(err, values); -const concatFiles = (files) => (callback) => { +const concatFiles = (files, iteratee) => (callback) => { log.trace('Resolved to the following files:'); files.forEach((file) => log.trace(file)); return async - .concat(files, csvToJson, concatFilesCb(callback)); + .concat(files, iteratee, concatFilesCb(callback)); }; module.exports = { concatFiles, concatFilesCb }; diff --git a/lib/merge-convert-csv-to-json.js b/lib/merge-convert-csv-to-json.js index e52e152..668547e 100644 --- a/lib/merge-convert-csv-to-json.js +++ b/lib/merge-convert-csv-to-json.js @@ -1,19 +1,31 @@ +/* eslint-disable global-require */ +/* eslint-disable import/no-dynamic-require */ const async = require('async'); const glob = require('glob'); +const fs = require('fs'); const { keyBy } = require('./keyBy'); const { getLogger } = require('./logger'); const { concatFiles } = require('./concatFiles'); const { writeToFile } = require('./writeToFile'); const { assignToProperty } = require('./assignToProperty'); - +const { csvToJson } = require('./csvToJson'); const log = getLogger('merge'); const mergeConvertCsv2Json = (options) => { const files = glob.sync(options.pattern); log.trace('Enabling ConcatFiles Modules'); - const modules = [concatFiles(files)]; + + const transformers = []; + if (options.transformerConfig) { + const transformerConfigAsString = fs.readFileSync(options.transformerConfig, 'utf8'); + const transformerConfig = JSON.parse(transformerConfigAsString); + transformers.push(...transformerConfig.map((config) => require(`./${config.pluginType}`)[config.pluginType].bind(null, config))); + } + const csvToJsonWithTransformers = async.compose(...transformers, csvToJson); + + const modules = [concatFiles(files, csvToJsonWithTransformers)]; if (options.keyBy) { log.trace(`Enabling KeyBy module for the option ${options.keyBy}`); diff --git a/lib/mergeGroups.js b/lib/mergeGroups.js index aa13cab..0309695 100644 --- a/lib/mergeGroups.js +++ b/lib/mergeGroups.js @@ -1,13 +1,14 @@ -const async = require('async'); +// const async = require('async'); -const { concatFiles } = require('./concatFiles'); -const { keyBy } = require('./keyBy'); -const { writeToFile } = require('./writeToFile'); +// const { concatFiles } = require('./concatFiles'); +// const { keyBy } = require('./keyBy'); +// const { writeToFile } = require('./writeToFile'); +// const { csvToJson } = require('./csvToJson'); -const mergeGroups = (groupFiles, mergedGroupsFilePath, cb) => async.waterfall([ - concatFiles(groupFiles), - keyBy('group_name'), - writeToFile(mergedGroupsFilePath), -], cb); +// const mergeGroups = (groupFiles, mergedGroupsFilePath, cb) => async.waterfall([ +// concatFiles(groupFiles, csvToJson), +// keyBy('group_name'), +// writeToFile(mergedGroupsFilePath), +// ], cb); -module.exports = { mergeGroups }; +// module.exports = { mergeGroups }; diff --git a/lib/mergeUsers.js b/lib/mergeUsers.js index eac0830..ab207b7 100644 --- a/lib/mergeUsers.js +++ b/lib/mergeUsers.js @@ -1,13 +1,14 @@ -const async = require('async'); +// const async = require('async'); -const { concatFiles } = require('./concatFiles'); -const { keyBy } = require('./keyBy'); -const { writeToFile } = require('./writeToFile'); +// const { concatFiles } = require('./concatFiles'); +// const { keyBy } = require('./keyBy'); +// const { writeToFile } = require('./writeToFile'); +// const { csvToJson } = require('./csvToJson'); -const mergeUsers = (userFiles, mergedUsersFilePath, cb) => async.waterfall([ - concatFiles(userFiles), - keyBy('email'), - writeToFile(mergedUsersFilePath), -], cb); +// const mergeUsers = (userFiles, mergedUsersFilePath, cb) => async.waterfall([ +// concatFiles(userFiles, csvToJson), +// keyBy('email'), +// writeToFile(mergedUsersFilePath), +// ], cb); -module.exports = { mergeUsers }; +// module.exports = { mergeUsers }; diff --git a/lib/transformPropertyToMap.js b/lib/transformPropertyToMap.js new file mode 100644 index 0000000..77be58a --- /dev/null +++ b/lib/transformPropertyToMap.js @@ -0,0 +1,24 @@ +const transformStringToMap = (config, jsonData) => { + const { + field, + keyIndex, + valueIndex, + propertySplitter, + keyValueSplitter, + } = config; + const fieldValue = jsonData[field]; + const transformedProperty = {}; + transformedProperty[field] = fieldValue.split(propertySplitter).reduce((acc, value) => { + const keyValuePair = value.split(keyValueSplitter); + acc[keyValuePair[keyIndex]] = keyValuePair[valueIndex]; + return acc; + }, {}); + return Object.assign(jsonData, transformedProperty); +}; + +const transformPropertyToMap = (config, data, callback) => { + const transformedData = data.map(transformStringToMap.bind(null, config)); + return callback(null, transformedData); +}; + +module.exports = { transformPropertyToMap };