Merge pull request #3 from nhsuk/feature/error-details

ℹ️ add error details to summary, tidy summary output

.travis.yml

@@ -1,5 +1,5 @@
 language: node_js
-node_js: node
+node_js: "9"
 cache: yarn
 
 before_install: yarn global add greenkeeper-lockfile@1

CHANGELOG.md

@@ -1,3 +1,7 @@
+0.2.0 / 2018-04-26
+==================
+- Add full error details to summary
+
 0.1.1 / 2018-03-19
 ==================
 - Expose output filename as env var

lib/etlStore.js

@@ -100,13 +100,15 @@ function saveRecords() {
 }
 
 function saveSummary() {
+  /* eslint-disable sort-keys */
   const summary = {
-    erroredIds: getErorredIds(),
     lastWritten: (new Date()).toLocaleString(),
-    totalErrored: getErorredIds().length,
-    totalFailed: erroredIds.length,
     totalScanned: ids.length,
+    totalErrored: getErorredIds().length,
+    erroredIds,
   };
+  /* eslint-enable sort-keys */
+
   fsHelper.saveJsonSync(summary, config.summaryFilename);
 }
 

lib/queues/populateRecordsFromIds.js

@@ -5,6 +5,8 @@ const etlStore = require('../etlStore');
 const limiter = require('../utils/limiter');
 const config = require('../config');
 
+const area = 'populateRecord';
+
 let populateRecordFromIdAction;
 let hitsPerWorker;
 let count = 0;
@@ -13,7 +15,7 @@ let totalRetries = 0;
 let saveEvery;
 
 function handleError(err, id) {
-  etlStore.addFailedId(id, err.message);
+  etlStore.addFailedId(id, area, err.message);
   log.error(`Error processing ID ${id}: ${err}`);
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "etl-toolkit",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "general components for managing the retrieval and processing of data",
   "main": "etlToolkit.js",
   "repository": "https://github.com/nhsuk/etl-toolkit",