Added integration-tests to test the entire flow

integration-tests/importedEntitiesFromPlayer.test.js

@@ -0,0 +1,111 @@
+const fs = require('fs');
+const path = require('path');
+const async = require('async');
+const csv = require('fast-csv');
+const should = require('should');
+
+const {normalize} = require('../sro_utils');
+const {findNodes} = require('../neo4j_utils');
+const log = require('../sro_utils/logger')("ImportedEntitiesFromPlayer");
+const csvOptions = {
+    objectMode: true,
+    headers: true
+};
+
+let n = "IBM Open Cloud - IBM Cloud Computing - United States"
+
+log.debug(n.normalize());
+
+let conceptRecordsNotFound = [];
+let contentRecordsNotFound = [];
+let      = [];
+let totalConceptNodes = 0;
+let totalContentNodes = 0;
+let totalCleanedContentNodes = 0;
+
+describe('Integration Test', () => {
+    it('Should have all the concept nodes imported from the csv file using perceptron player', (done) => {
+        const findNodeOperations = [];
+        csv.fromPath(path.join(__dirname,'../data/cloud_concepts_final.csv'), csvOptions)
+        .on('data', (data) => {
+            const node = {
+                label: 'concept',
+                name: data.Name.normalize()
+            };
+            totalConceptNodes++;
+            findNodeOperations.push(findNodes.bind(null, node));
+        })
+        .on('end', () => {
+            async.parallel(findNodeOperations, (err, results) => {
+                results.forEach((result) => {
+                    if(result.records.length < 1) {
+                        conceptRecordsNotFound.push(result);
+                    }
+                });
+                // log.debug({records:conceptRecordsNotFound});
+                fs.writeFileSync(path.join(__dirname, './notFoundConceptNodes.json'), JSON.stringify(conceptRecordsNotFound));
+                log.debug(`Total Concept Nodes ${totalConceptNodes}`);
+                log.debug(`Concept Nodes Not Found in Graph ${conceptRecordsNotFound.length}`);
+                done();
+            });
+        });
+    });
+
+    it('Should have all the content nodes imported from the csv file using perceptron player', (done) => {
+        const findNodeOperations = [];
+        csv.fromPath(path.join(__dirname,'../data/cloud course_1423743638000.csv'), csvOptions)
+        .on('data', (data) => {
+            if(undefined !== typeof data.name && data['node type'] === 'content' ) {
+                const node = {
+                    label: 'content',
+                    name: data.name.normalize()
+                };
+                totalContentNodes++;
+                findNodeOperations.push(findNodes.bind(null, node));
+            }
+        })
+        .on('end', () => {
+            async.parallel(findNodeOperations, (err, results) => {
+                results.forEach((result) => {
+                    if(result.records.length < 1) {
+                        contentRecordsNotFound.push(result);
+                    }
+                });
+                fs.writeFileSync(path.join(__dirname, './notFoundContentNodes.json'), JSON.stringify(contentRecordsNotFound));
+                log.debug(`Total Content Nodes ${totalContentNodes}`);
+                log.debug(`Content Nodes Not Found in Graph ${contentRecordsNotFound.length}`);
+                done();
+            });
+        });
+    });
+
+    it('Should have all the cleaned content nodes imported from the csv file using perceptron player', (done) => {
+        const findNodeOperations = [];
+        csv.fromPath(path.join(__dirname,'../data/cloud_course_results_with_summaries_cleanedup.csv'), csvOptions)
+        .on('data', (data) => {
+            if(undefined !== typeof data.name && data['node type'] === 'Content' ) {
+                const node = {
+                    label: 'content',
+                    name: data.name.normalize()
+                };
+                totalCleanedContentNodes++;
+                findNodeOperations.push(findNodes.bind(null, node));
+            }
+        })
+        .on('end', () => {
+            async.parallel(findNodeOperations, (err, results) => {
+                results.forEach((result) => {
+                    if(result.records.length < 1) {
+                        cleanedContentRecordsNotFound.push(result);
+                    }
+                });
+                fs.writeFileSync(path.join(__dirname, './notFoundCleanedContentNodes.json'), JSON.stringify(contentRecordsNotFound));
+                log.debug(`Total Content Nodes ${totalCleanedContentNodes}`);
+                log.debug(`Content Nodes Not Found in Graph ${cleanedContentRecordsNotFound.length}`);
+                done();
+            });
+        });
+    });
+});
+
+

integration-tests/notFoundConceptNodes.json

@@ -0,0 +1 @@
+[{"records":[],"summary":{"statement":{"text":"MATCH (n:concept {name:\"server_virtualization\"}) return n","parameters":{}},"statementType":"r","counters":{"_stats":{"nodesCreated":0,"nodesDeleted":0,"relationshipsCreated":0,"relationshipsDeleted":0,"propertiesSet":0,"labelsAdded":0,"labelsRemoved":0,"indexesAdded":0,"indexesRemoved":0,"constraintsAdded":0,"constraintsRemoved":0}},"updateStatistics":{"_stats":{"nodesCreated":0,"nodesDeleted":0,"relationshipsCreated":0,"relationshipsDeleted":0,"propertiesSet":0,"labelsAdded":0,"labelsRemoved":0,"indexesAdded":0,"indexesRemoved":0,"constraintsAdded":0,"constraintsRemoved":0}},"plan":false,"profile":false,"notifications":[],"server":{"address":"localhost:7687","version":"Neo4j/3.2.0"},"resultConsumedAfter":{"low":1,"high":0},"resultAvailableAfter":{"low":1,"high":0}}},{"records":[],"summary":{"statement":{"text":"MATCH (n:concept {name:\"Private_Cloud\"}) return n","parameters":{}},"statementType":"r","counters":{"_stats":{"nodesCreated":0,"nodesDeleted":0,"relationshipsCreated":0,"relationshipsDeleted":0,"propertiesSet":0,"labelsAdded":0,"labelsRemoved":0,"indexesAdded":0,"indexesRemoved":0,"constraintsAdded":0,"constraintsRemoved":0}},"updateStatistics":{"_stats":{"nodesCreated":0,"nodesDeleted":0,"relationshipsCreated":0,"relationshipsDeleted":0,"propertiesSet":0,"labelsAdded":0,"labelsRemoved":0,"indexesAdded":0,"indexesRemoved":0,"constraintsAdded":0,"constraintsRemoved":0}},"plan":false,"profile":false,"notifications":[],"server":{"address":"localhost:7687","version":"Neo4j/3.2.0"},"resultConsumedAfter":{"low":1,"high":0},"resultAvailableAfter":{"low":1,"high":0}}}]

integration-tests/package.json

@@ -0,0 +1,12 @@
+{
+  "devDependencies": {
+    "should": "^11.2.1"
+  },
+  "name": "integration-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "fast-csv": "^2.4.0"
+  }
+}

integration-tests/yarn.lock

@@ -0,0 +1,111 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+arguments-extended@~0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/arguments-extended/-/arguments-extended-0.0.3.tgz#6107e4917d0eb6f0a4dd66320fc15afc72ef4946"
+  dependencies:
+    extended "~0.0.3"
+    is-extended "~0.0.8"
+
+array-extended@~0.0.3, array-extended@~0.0.4, array-extended@~0.0.5:
+  version "0.0.11"
+  resolved "https://registry.yarnpkg.com/array-extended/-/array-extended-0.0.11.tgz#d7144ae748de93ca726f121009dbff1626d164bd"
+  dependencies:
+    arguments-extended "~0.0.3"
+    extended "~0.0.3"
+    is-extended "~0.0.3"
+
+date-extended@~0.0.3:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/date-extended/-/date-extended-0.0.6.tgz#23802d57dd1bf7818813fe0c32e851a86da267c9"
+  dependencies:
+    array-extended "~0.0.3"
+    extended "~0.0.3"
+    is-extended "~0.0.3"
+
+declare.js@~0.0.4:
+  version "0.0.8"
+  resolved "https://registry.yarnpkg.com/declare.js/-/declare.js-0.0.8.tgz#0478adff9564c004f51df73d8bc134019d28dcde"
+
+extended@~0.0.3, extended@0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/extended/-/extended-0.0.6.tgz#7fb8bf7b9dae397586e48570acfd642c78e50669"
+  dependencies:
+    extender "~0.0.5"
+
+extender@~0.0.5:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/extender/-/extender-0.0.10.tgz#589c07482be61a1460b6d81f9c24aa67e8f324cd"
+  dependencies:
+    declare.js "~0.0.4"
+
+fast-csv:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/fast-csv/-/fast-csv-2.4.0.tgz#f6e93185b6d290e5808948869c3fd09f64fbc842"
+  dependencies:
+    extended "0.0.6"
+    is-extended "0.0.10"
+    object-extended "0.0.7"
+    string-extended "0.0.8"
+
+is-extended@~0.0.3, is-extended@~0.0.8, is-extended@0.0.10:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/is-extended/-/is-extended-0.0.10.tgz#244e140df75bb1c9a3106f412ff182fb534a6d62"
+  dependencies:
+    extended "~0.0.3"
+
+object-extended@0.0.7:
+  version "0.0.7"
+  resolved "https://registry.yarnpkg.com/object-extended/-/object-extended-0.0.7.tgz#84fd23f56b15582aeb3e88b05cb55d2432d68a33"
+  dependencies:
+    array-extended "~0.0.4"
+    extended "~0.0.3"
+    is-extended "~0.0.3"
+
+should:
+  version "11.2.1"
+  resolved "https://registry.yarnpkg.com/should/-/should-11.2.1.tgz#90f55145552d01cfc200666e4e818a1c9670eda2"
+  dependencies:
+    should-equal "^1.0.0"
+    should-format "^3.0.2"
+    should-type "^1.4.0"
+    should-type-adaptors "^1.0.1"
+    should-util "^1.0.0"
+
+should-equal@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/should-equal/-/should-equal-1.0.1.tgz#0b6e9516f2601a9fb0bb2dcc369afa1c7e200af7"
+  dependencies:
+    should-type "^1.0.0"
+
+should-format@^3.0.2:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/should-format/-/should-format-3.0.3.tgz#9bfc8f74fa39205c53d38c34d717303e277124f1"
+  dependencies:
+    should-type "^1.3.0"
+    should-type-adaptors "^1.0.1"
+
+should-type-adaptors@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/should-type-adaptors/-/should-type-adaptors-1.0.1.tgz#efe5553cdf68cff66e5c5f51b712dc351c77beaa"
+  dependencies:
+    should-type "^1.3.0"
+    should-util "^1.0.0"
+
+should-type@^1.0.0, should-type@^1.3.0, should-type@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/should-type/-/should-type-1.4.0.tgz#0756d8ce846dfd09843a6947719dfa0d4cff5cf3"
+
+should-util@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/should-util/-/should-util-1.0.0.tgz#c98cda374aa6b190df8ba87c9889c2b4db620063"
+
+string-extended@0.0.8:
+  version "0.0.8"
+  resolved "https://registry.yarnpkg.com/string-extended/-/string-extended-0.0.8.tgz#741957dff487b0272a79eec5a44f239ee6f17ccd"
+  dependencies:
+    array-extended "~0.0.5"
+    date-extended "~0.0.3"
+    extended "~0.0.3"
+    is-extended "~0.0.3"
+