diff --git a/.gitignore b/.gitignore index a491f57..71704ad 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,7 @@ typings/ /db # Code Editor Configuration file -.vscode \ No newline at end of file +.vscode + +# CSV test file +assessmentItem.csv \ No newline at end of file diff --git a/app.js b/app.js index 2d8beb9..8cf761f 100644 --- a/app.js +++ b/app.js @@ -6,6 +6,7 @@ const bunyanMiddleware = require('bunyan-middleware'); const ping = require('./ping'); const question = require('./question'); const history = require('./history'); +const bulkUpload = require('./bulk_upload'); const app = express(); const logger = bunyan.createLogger({ name: 'Question-Authoring-API' }); @@ -31,5 +32,6 @@ app.use(bunyanMiddleware({ app.use('/ping', ping); app.use('/history', history); app.use('/question', question); +app.use('/bulkupload', bulkUpload); module.exports = app; diff --git a/bulk_upload/csv.controller.js b/bulk_upload/csv.controller.js new file mode 100644 index 0000000..d3941cf --- /dev/null +++ b/bulk_upload/csv.controller.js @@ -0,0 +1,42 @@ +const questionController = require('../question/question.controller'); +const config = require('../config'); +const fs = require('fs'); +const csv = require('fast-csv'); +const _ = require('lodash'); + +const readAndPublish = async () => { + const stream = fs.createReadStream(config.fileUploadPath); + let publishedQuestion = ''; + await csv + .fromStream(stream, { objectMode: true, headers: true }) + .validate(data => + data.concept && data.content && data.expectedOutcome && data.player && data.evaluator) + .on('data-invalid', () => { + console.log('Error: Primary fields are missing!!!'); + }) + .on('data', async (data) => { + let assessmentItem = data; + let primaryObject = {}; + primaryObject = _.pick(assessmentItem, ['concept', 'content', 'expectedOutcome', 'player', 'evaluator']); + _.forOwnRight(assessmentItem, (value, key) => { + if (key === 'concept' || key === 'content' || key === 'expectedOutcome' || key === 'player' || key === 'evaluator') { + _.unset(assessmentItem, key); + } + }); + const secondaryObject = assessmentItem; + primaryObject.question = secondaryObject; + assessmentItem = primaryObject; + const initialisedQuestion = await questionController.initQuestion(); + assessmentItem.id = initialisedQuestion.id; + publishedQuestion = await questionController.publishQuestion(assessmentItem); + return publishedQuestion; + }) + .on('error', (err) => { + console.log(`Error is ${err}`); + }); +}; + +module.exports = { + readAndPublish, +}; + diff --git a/bulk_upload/csv.router.js b/bulk_upload/csv.router.js new file mode 100644 index 0000000..b05563a --- /dev/null +++ b/bulk_upload/csv.router.js @@ -0,0 +1,11 @@ +const express = require('express'); +const csvController = require('./csv.controller'); + +const router = express.Router(); + +router.get('/csvUpload', (req, res) => { + const publishedQuestion = csvController.readAndPublish(); + res.json(publishedQuestion); +}); + +module.exports = router; diff --git a/bulk_upload/index.js b/bulk_upload/index.js new file mode 100644 index 0000000..1efbac5 --- /dev/null +++ b/bulk_upload/index.js @@ -0,0 +1 @@ +module.exports = require('./csv.router'); diff --git a/config/config.js b/config/config.js index 67a688b..6e93bcc 100644 --- a/config/config.js +++ b/config/config.js @@ -1,6 +1,7 @@ const config = { PORT: process.env.PORT || 4000, MONGODB_URL: process.env.MONGODB_URL || 'localhost:29017/assessment_item', + fileUploadPath: process.env.FILE_UPLOAD_PATH || 'assessmentItem.csv', }; module.exports = config; diff --git a/package-lock.json b/package-lock.json index 521c04c..3e9c2ac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -803,8 +803,7 @@ "diff": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", - "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", - "dev": true + "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==" }, "doctrine": { "version": "2.0.0", @@ -1585,6 +1584,14 @@ "mime-types": "2.1.17" } }, + "formatio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.2.0.tgz", + "integrity": "sha1-87IWfZBoxGmKjVH092CjmlTYGOs=", + "requires": { + "samsam": "1.3.0" + } + }, "formidable": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.1.1.tgz", @@ -2669,6 +2676,11 @@ "integrity": "sha1-OGchPo3Xm/Ho8jAMDPwe+xgsDfE=", "dev": true }, + "just-extend": { + "version": "1.1.27", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-1.1.27.tgz", + "integrity": "sha512-mJVp13Ix6gFo3SBAy9U/kL+oeZqzlYYYLQBwXVBlVzIsZwBqGREnOro24oC/8s8aox+rJhtZ2DiQof++IrkA+g==" + }, "kareem": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/kareem/-/kareem-1.5.0.tgz", @@ -2846,6 +2858,11 @@ "lodash._root": "3.0.1" } }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" + }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -2926,6 +2943,11 @@ "integrity": "sha1-euTsJXMC/XkNVXyxDJcQDYV7AFY=", "dev": true }, + "lolex": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-2.1.3.tgz", + "integrity": "sha512-BdHq78SeI+6PAUtl4atDuCt7L6E4fab3mSRtqxm4ywaXe4uP7jZ0TTcFNuU20syUjxZc2l7jFqKVMJ+AX0LnpQ==" + }, "loose-envify": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.3.1.tgz", @@ -3241,6 +3263,38 @@ "babel-runtime": "6.26.0" } }, + "nise": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-1.2.0.tgz", + "integrity": "sha512-q9jXh3UNsMV28KeqI43ILz5+c3l+RiNW8mhurEwCKckuHQbL+hTJIKKTiUlCPKlgQ/OukFvSnKB/Jk3+sFbkGA==", + "requires": { + "formatio": "1.2.0", + "just-extend": "1.1.27", + "lolex": "1.6.0", + "path-to-regexp": "1.7.0", + "text-encoding": "0.6.4" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "lolex": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.6.0.tgz", + "integrity": "sha1-OpoCg0UqR9dDnnJzG54H1zhuSfY=" + }, + "path-to-regexp": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz", + "integrity": "sha1-Wf3g9DW62suhA6hOnTvGTpa5k30=", + "requires": { + "isarray": "0.0.1" + } + } + } + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -5606,6 +5660,11 @@ "integrity": "sha1-gaCY9Efku8P/MxKiQ1IbwGDvWRE=", "optional": true }, + "samsam": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz", + "integrity": "sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==" + }, "semver": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", @@ -5759,6 +5818,30 @@ } } }, + "sinon": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-4.0.2.tgz", + "integrity": "sha512-4mUsjHfjrHyPFGDTtNJl0q8cv4VOJGvQykI1r3fnn05ys0sQL9M1Y+DyyGNWLD2PMcoyqjJ/nFDm4K54V1eQOg==", + "requires": { + "diff": "3.3.1", + "formatio": "1.2.0", + "lodash.get": "4.4.2", + "lolex": "2.1.3", + "nise": "1.2.0", + "supports-color": "4.5.0", + "type-detect": "4.0.3" + }, + "dependencies": { + "supports-color": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.5.0.tgz", + "integrity": "sha1-vnoN5ITexcXN34s9WRJQRJEvY1s=", + "requires": { + "has-flag": "2.0.0" + } + } + } + }, "slice-ansi": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz", @@ -5968,6 +6051,11 @@ "string-width": "2.1.1" } }, + "text-encoding": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", + "integrity": "sha1-45mpgiV6J22uQou5KEXLcb3CbRk=" + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -6049,6 +6137,11 @@ "prelude-ls": "1.1.2" } }, + "type-detect": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.3.tgz", + "integrity": "sha1-Dj8mcLRAmbC0bChNE2p+9Jx0wuo=" + }, "type-is": { "version": "1.6.15", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.15.tgz", diff --git a/package.json b/package.json index 4cc4878..9d9a164 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "bunyan-middleware": "^0.8.0", "eslint": "^4.9.0", "express": "^4.16.2", + "fast-csv": "^2.4.1", "json-diff-rfc6902": "^1.3.3", + "lodash": "^4.17.4", "mongoose": "^4.12.4", "simple-neo4j-wrapper": "^1.0.5", "sinon": "^4.0.1"