Merge pull request #5 from nishant-jain-94/dev

Added plugin to transform value of type string to map with supporting config file

CHANGELOG.md

@@ -1,16 +1,26 @@
 # Changelog
 
-## v1.0.0
+## v1.1.0
 
-#### :rocket: Features
+#### :rocket: Added Features
+
+- Added options to convert transform string values into map using a pluginType named `transformPropertyToMap`
+
+#### :fire: Removed
+- Removed files `mergeUsers.js` and `mergeGroups.js` which was being unused
 
-- Merge all the `CSV` files and convert into single `JSON` file.
-- Merge all the `CSV` files, convert it to `JSON` object, assign it to a `JSON field` an then store it in a `JSON` file.
-- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field and then store it to a `JSON` file.
-- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field, assign the object to a `JSON` field and then store it to a `JSON` file.
 
 ## v1.0.1
 
 #### :bug: Bug Fixes
 
 - Removed the code which was trying to read a file which didn't existed, a junk which creeped during the development cycle.
+
+## v1.0.0
+
+#### :rocket: Features
+
+- Merge all the `CSV` files and convert into single `JSON` file.
+- Merge all the `CSV` files, convert it to `JSON` object, assign it to a `JSON field` an then store it in a `JSON` file.
+- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field and then store it to a `JSON` file.
+- Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field, assign the object to a `JSON` field and then store it to a `JSON` file.

README.md

@@ -10,7 +10,7 @@ Merges all the csv files and converts it to json.
 - Merge all the `CSV` files, convert it to `JSON` object, assign it to a `JSON field` an then store it in a `JSON` file.
 - Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field and then store it to a `JSON` file.
 - Merge all the `CSV` files, convert it to `JSON` with every object *keyedBy* a `JSON` field, assign the object to a `JSON` field and then store it to a `JSON` file.
-
+- Merge all the `CSV` files, convert into single `JSON` file by converting string values into map by providing `keyValueSplitter`, `propertySplitter`, `keyIndex`, `valueIndex`, `field` and `pluginType` as `transfromPropertyToMap`
 
 ## Installation
 
@@ -53,3 +53,29 @@ yarn add merge-convert-csv-to-json
   ```
   merge-convert --pattern "/users/*.users.csv" --keyBy email --assignTo users --outputTo /users/merged.users.json
   ```
+- Merge all the `CSV` files, convert into single `JSON` file by converting string values into map by providing `keyValueSplitter`, `propertySplitter`, `keyIndex`, `valueIndex`, `field` and `pluginType` as `transfromPropertyToMap`
+  ```json
+  # transformer-config.json
+  [
+    {
+      pluginType: "transformPropertyToMap",
+      keyValueSplitter: ":",
+      propertySplitter: ",",
+      keyIndex: "0",
+      valueIndex: "1",
+      field: "groups_access"
+    },
+    {
+      pluginType: "transformPropertyToMap",
+      keyValueSplitter: ":",
+      propertySplitter: ",",
+      keyIndex: "0",
+      valueIndex: "1",
+      field: "project_access"
+    }
+  ]
+  ```
+
+  ```
+  merge-convert --pattern "/users/*.users.csv" --outputTo /users/merged.users.json --transformerConfig transformer-config.json
+  ```

__tests__/bin/merge-convert.js

@@ -5,15 +5,21 @@ const fs = require('fs');
 beforeEach(() => {
   const sampleA = 'username,email,organization,password\ntommy.jones,tommy.jones@northwindtraders.com,NorthwindTraders,password@123\n';
   const sampleB = 'username,email,organization,password\nanne.frank,anne.frank@northwindtraders.com,NorthwindTraders,password@123\n';
+  const sampleC = 'username,email,organization,password,groups_access,project_access\nanne.frank,anne.frank@northwindtraders.com,NorthwindTraders,password@123,"northwind-wave-1:guest,northwind-auditors:guest","northwind-wave-1:guest,northwind-auditors:guest"';
+  const transformerConfig = '[{"pluginType":"transformPropertyToMap","keyValueSplitter":":","propertySplitter":",","keyIndex":"0","valueIndex":"1","field":"groups_access"},{"pluginType":"transformPropertyToMap","keyValueSplitter":":","propertySplitter":",","keyIndex":"0","valueIndex":"1","field":"project_access"}]';
 
   fs.writeFileSync('/tmp/a.users.csv', sampleA);
   fs.writeFileSync('/tmp/b.users.csv', sampleB);
+  fs.writeFileSync('/tmp/c.groups.csv', sampleC);
+  fs.writeFileSync('/tmp/transformerConfig.json', transformerConfig);
 });
 
 afterEach(() => {
   fs.unlinkSync('/tmp/a.users.csv');
   fs.unlinkSync('/tmp/b.users.csv');
   fs.unlinkSync('/tmp/merged.users.json');
+  fs.unlinkSync('/tmp/c.groups.csv');
+  fs.unlinkSync('/tmp/transformerConfig.json');
 });
 
 
@@ -124,3 +130,33 @@ test('Should Merge all the `CSV` files, convert it to `JSON` with every object *
     done();
   });
 });
+
+test('Should merge all the CSV files and convert into single JSON file with Transformers', (done) => {
+  const pathToMergeConvert = path.join(__dirname, '../../bin/merge-convert.js');
+  const cmd = `${pathToMergeConvert} --pattern "/tmp/c.groups.csv" --outputTo /tmp/merged.users.json --transformerConfig /tmp/transformerConfig.json`;
+  const expected = [{
+    username: 'anne.frank',
+    email: 'anne.frank@northwindtraders.com',
+    organization: 'NorthwindTraders',
+    password: 'password@123',
+    groups_access: {
+      'northwind-wave-1': 'guest',
+      'northwind-auditors': 'guest',
+    },
+    project_access: {
+      'northwind-wave-1': 'guest',
+      'northwind-auditors': 'guest',
+    },
+  }];
+  childProcess.exec(cmd, (err, stdout, stderr) => {
+    if (!err) {
+      const mergedJSONAsString = fs.readFileSync('/tmp/merged.users.json', 'utf8');
+      const mergedJSON = JSON.parse(mergedJSONAsString);
+      expect(mergedJSON).toMatchObject(expected);
+    } else {
+      console.log(stdout);
+      console.log(stderr);
+    }
+    done();
+  });
+});

__tests__/lib/concatFiles.js

@@ -1,6 +1,7 @@
 const mock = require('mock-fs');
 
 const { concatFiles } = require('../../lib/concatFiles');
+const { csvToJson } = require('../../lib/csvToJson');
 
 beforeAll(() => mock({
   '/tmp/a.csv': 'username,email,organization,password\ntommy.jones,tommy.jones@northwindtraders.com,NorthwindTraders,password@123\n',
@@ -26,14 +27,15 @@ test('Given set of csv files it should return the merged json', (done) => {
     expect(values).toEqual(expect.arrayContaining(expected));
     done(err);
   };
-  concatFiles(['/tmp/a.csv', '/tmp/b.csv'])(testCb);
+  concatFiles(['/tmp/a.csv', '/tmp/b.csv'], csvToJson)(testCb);
 });
 
 test('Given a non existing path it should throw error', (done) => {
   const testCb = (err) => {
     if (err) done();
   };
-  concatFiles(['/tmp/d.csv'])(testCb);
+
+  concatFiles(['/tmp/d.csv'], csvToJson)(testCb);
 });
 
 afterAll(async () => {

__tests__/lib/transformPropertyToMap.js

@@ -0,0 +1,39 @@
+
+const { transformPropertyToMap } = require('../../lib/transformPropertyToMap.js');
+
+test('Should be able to convert value of type string to map', (done) => {
+  const transformerConfig = {
+    pluginType: 'transformPropertyToMap',
+    keyValueSplitter: ':',
+    propertySplitter: ',',
+    keyIndex: '0',
+    valueIndex: '1',
+    field: 'groups_access',
+  };
+
+  const data = [
+    {
+      username: 'john.doe',
+      email: 'john.doe@northwind.in',
+      password: 'password@123',
+      groups_access: 'northwind-group:guest,northwind-auditors:guest',
+    },
+  ];
+
+  const expected = [
+    {
+      username: 'john.doe',
+      email: 'john.doe@northwind.in',
+      password: 'password@123',
+      groups_access: {
+        'northwind-group': 'guest',
+        'northwind-auditors': 'guest',
+      },
+    },
+  ];
+
+  transformPropertyToMap(transformerConfig, data, (err, transformedData) => {
+    expect(transformedData).toMatchObject(expected);
+    done();
+  });
+});

bin/merge-convert.js

@@ -10,6 +10,7 @@ const optionDefinitions = [
   { name: 'keyBy', type: String },
   { name: 'assignTo', type: String },
   { name: 'outputTo', type: String },
+  { name: 'transformerConfig', type: String },
 ];
 
 try {

lib/cli-usage.js

@@ -28,6 +28,11 @@ const sections = [
         typeLabel: '{underline filePath}',
         description: 'The filepath to output the result to.',
       },
+      {
+        name: 'transformerConfig',
+        typeLabel: '{underline propertyName}',
+        description: 'The name of the file containing the config for transformation',
+      },
     ],
   },
 ];

lib/concatFiles.js

@@ -1,17 +1,16 @@
 const async = require('async');
 
 const { getLogger } = require('./logger');
-const { csvToJson } = require('./csvToJson');
 
 const log = getLogger('concatFiles');
 
 const concatFilesCb = (callback) => (err, values) => callback(err, values);
 
-const concatFiles = (files) => (callback) => {
+const concatFiles = (files, iteratee) => (callback) => {
   log.trace('Resolved to the following files:');
   files.forEach((file) => log.trace(file));
   return async
-    .concat(files, csvToJson, concatFilesCb(callback));
+    .concat(files, iteratee, concatFilesCb(callback));
 };
 
 module.exports = { concatFiles, concatFilesCb };

lib/merge-convert-csv-to-json.js

@@ -1,19 +1,31 @@
+/* eslint-disable global-require */
+/* eslint-disable import/no-dynamic-require */
 const async = require('async');
 const glob = require('glob');
+const fs = require('fs');
 
 const { keyBy } = require('./keyBy');
 const { getLogger } = require('./logger');
 const { concatFiles } = require('./concatFiles');
 const { writeToFile } = require('./writeToFile');
 const { assignToProperty } = require('./assignToProperty');
-
+const { csvToJson } = require('./csvToJson');
 
 const log = getLogger('merge');
 
 const mergeConvertCsv2Json = (options) => {
   const files = glob.sync(options.pattern);
   log.trace('Enabling ConcatFiles Modules');
-  const modules = [concatFiles(files)];
+
+  const transformers = [];
+  if (options.transformerConfig) {
+    const transformerConfigAsString = fs.readFileSync(options.transformerConfig, 'utf8');
+    const transformerConfig = JSON.parse(transformerConfigAsString);
+    transformers.push(...transformerConfig.map((config) => require(`./${config.pluginType}`)[config.pluginType].bind(null, config)));
+  }
+  const csvToJsonWithTransformers = async.compose(...transformers, csvToJson);
+
+  const modules = [concatFiles(files, csvToJsonWithTransformers)];
 
   if (options.keyBy) {
     log.trace(`Enabling KeyBy module for the option ${options.keyBy}`);

lib/mergeGroups.js

@@ -1,13 +1,14 @@
-const async = require('async');
+// const async = require('async');
 
-const { concatFiles } = require('./concatFiles');
-const { keyBy } = require('./keyBy');
-const { writeToFile } = require('./writeToFile');
+// const { concatFiles } = require('./concatFiles');
+// const { keyBy } = require('./keyBy');
+// const { writeToFile } = require('./writeToFile');
+// const { csvToJson } = require('./csvToJson');
 
-const mergeGroups = (groupFiles, mergedGroupsFilePath, cb) => async.waterfall([
-  concatFiles(groupFiles),
-  keyBy('group_name'),
-  writeToFile(mergedGroupsFilePath),
-], cb);
+// const mergeGroups = (groupFiles, mergedGroupsFilePath, cb) => async.waterfall([
+//   concatFiles(groupFiles, csvToJson),
+//   keyBy('group_name'),
+//   writeToFile(mergedGroupsFilePath),
+// ], cb);
 
-module.exports = { mergeGroups };
+// module.exports = { mergeGroups };

lib/mergeUsers.js

@@ -1,13 +1,14 @@
-const async = require('async');
+// const async = require('async');
 
-const { concatFiles } = require('./concatFiles');
-const { keyBy } = require('./keyBy');
-const { writeToFile } = require('./writeToFile');
+// const { concatFiles } = require('./concatFiles');
+// const { keyBy } = require('./keyBy');
+// const { writeToFile } = require('./writeToFile');
+// const { csvToJson } = require('./csvToJson');
 
-const mergeUsers = (userFiles, mergedUsersFilePath, cb) => async.waterfall([
-  concatFiles(userFiles),
-  keyBy('email'),
-  writeToFile(mergedUsersFilePath),
-], cb);
+// const mergeUsers = (userFiles, mergedUsersFilePath, cb) => async.waterfall([
+//   concatFiles(userFiles, csvToJson),
+//   keyBy('email'),
+//   writeToFile(mergedUsersFilePath),
+// ], cb);
 
-module.exports = { mergeUsers };
+// module.exports = { mergeUsers };