From 2ce9816cbc75b525aed48ea01e12e2de2157376e Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 25 Aug 2020 11:24:45 +0530 Subject: [PATCH 1/7] Fix issue where migration script copied data from db before it could delete existing data in ES --- scripts/db/dumpDbToEs.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index e8218ee..17c51cf 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -11,15 +11,12 @@ const { async function cleanupES () { const client = getESClient() - client.indices.delete({ + + await client.indices.delete({ index: '_all' - }, function (err, res) { - if (err) { - console.error(err.message) - } else { - console.log('Existing indices have been deleted!') - } }) + + console.log('Existing indices have been deleted!') } async function insertIntoES (modelName, body) { From 9072ac5ce1c8502a462c5defc6e333ef4fb29f0a Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 25 Aug 2020 14:03:00 +0530 Subject: [PATCH 2/7] Debug code - get data from db to identify and remove invalid data --- src/models/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/models/index.js b/src/models/index.js index 8eecacc..07808e7 100755 --- a/src/models/index.js +++ b/src/models/index.js @@ -58,4 +58,10 @@ module.exports.init = async () => { await DBHelper.createTable(Attribute) await DBHelper.createTable(UserAttribute) await DBHelper.createTable(OrganizationSkillsProvider) + + // Debug code - to remove invalid entries from db + let data = DBHelper.find(ExternalProfile, []) + logger.info(console.log(data, null, 4)) + data = DBHelper.find(UserAttribute, []) + logger.info(console.log(data, null, 4)) } From d4ecc8642e4a2240ec3bc1bf7e6ded9e4ea190ae Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 25 Aug 2020 14:06:12 +0530 Subject: [PATCH 3/7] Debug code - to get data from db to identify and remove invalid data --- src/models/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/index.js b/src/models/index.js index 07808e7..f23ef23 100755 --- a/src/models/index.js +++ b/src/models/index.js @@ -60,8 +60,8 @@ module.exports.init = async () => { await DBHelper.createTable(OrganizationSkillsProvider) // Debug code - to remove invalid entries from db - let data = DBHelper.find(ExternalProfile, []) + let data = await DBHelper.find(ExternalProfile, []) logger.info(console.log(data, null, 4)) - data = DBHelper.find(UserAttribute, []) + data = await DBHelper.find(UserAttribute, []) logger.info(console.log(data, null, 4)) } From c2c1f84c730de40556f7e2dee29df4ec051753ae Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 25 Aug 2020 14:17:24 +0530 Subject: [PATCH 4/7] Debug completed - undoing debug code --- src/models/index.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/models/index.js b/src/models/index.js index f23ef23..8eecacc 100755 --- a/src/models/index.js +++ b/src/models/index.js @@ -58,10 +58,4 @@ module.exports.init = async () => { await DBHelper.createTable(Attribute) await DBHelper.createTable(UserAttribute) await DBHelper.createTable(OrganizationSkillsProvider) - - // Debug code - to remove invalid entries from db - let data = await DBHelper.find(ExternalProfile, []) - logger.info(console.log(data, null, 4)) - data = await DBHelper.find(UserAttribute, []) - logger.info(console.log(data, null, 4)) } From 6255940be098aff72b008b5579f42d3044d863a1 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Sat, 29 Aug 2020 19:54:00 +0530 Subject: [PATCH 5/7] Allow empty string for certifierId --- src/modules/achievement/service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/achievement/service.js b/src/modules/achievement/service.js index e57fcaf..61695f7 100644 --- a/src/modules/achievement/service.js +++ b/src/modules/achievement/service.js @@ -12,7 +12,7 @@ const methods = helper.getServiceMethods( achievementsProviderId: joi.string().required(), name: joi.string().required(), uri: joi.string(), - certifierId: joi.string(), + certifierId: joi.string().allow(), certifiedDate: joi.date().format('iso') }, { // patch request body joi schema @@ -20,7 +20,7 @@ const methods = helper.getServiceMethods( achievementsProviderId: joi.string().required(), name: joi.string(), uri: joi.string(), - certifierId: joi.string(), + certifierId: joi.string().allow(), certifiedDate: joi.date().format('iso') }, { // search request query joi schema From 8ba24610dec10a7b69b642d9645427dd7539c692 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Sat, 29 Aug 2020 20:01:04 +0530 Subject: [PATCH 6/7] Allow empty string for certifierId --- src/modules/achievement/service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/achievement/service.js b/src/modules/achievement/service.js index 61695f7..02109fa 100644 --- a/src/modules/achievement/service.js +++ b/src/modules/achievement/service.js @@ -12,7 +12,7 @@ const methods = helper.getServiceMethods( achievementsProviderId: joi.string().required(), name: joi.string().required(), uri: joi.string(), - certifierId: joi.string().allow(), + certifierId: joi.string().allow(''), certifiedDate: joi.date().format('iso') }, { // patch request body joi schema @@ -20,7 +20,7 @@ const methods = helper.getServiceMethods( achievementsProviderId: joi.string().required(), name: joi.string(), uri: joi.string(), - certifierId: joi.string().allow(), + certifierId: joi.string().allow(''), certifiedDate: joi.date().format('iso') }, { // search request query joi schema From fce1c636af3c913d74430df4bdab55218ece1cd2 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Sat, 29 Aug 2020 23:14:03 +0530 Subject: [PATCH 7/7] Post event to aggregate topic as well --- README.md | 1 + config/default.js | 1 + src/common/helper.js | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index d35fd17..827e445 100755 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_CREATE_TOPIC: Kafka topic for create message - UBAHN_UPDATE_TOPIC: Kafka topic for update message - UBAHN_DELETE_TOPIC: Kafka topic for delete message +- UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) - ES.HOST: Elasticsearch host - ES.API_VERSION: Elasticsearch API version - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. diff --git a/config/default.js b/config/default.js index 8ae692e..00b208f 100755 --- a/config/default.js +++ b/config/default.js @@ -36,6 +36,7 @@ module.exports = { UBAHN_CREATE_TOPIC: process.env.UBAHN_CREATE_TOPIC || 'u-bahn.action.create', UBAHN_UPDATE_TOPIC: process.env.UBAHN_UPDATE_TOPIC || 'u-bahn.action.update', UBAHN_DELETE_TOPIC: process.env.UBAHN_DELETE_TOPIC || 'u-bahn.action.delete', + UBAHN_AGGREGATE_TOPIC: process.env.UBAHN_AGGREGATE_TOPIC || 'u-bahn.action.aggregate', EMSI: { CLIENT_ID: process.env.EMSI_CLIENT_ID, diff --git a/src/common/helper.js b/src/common/helper.js index e6bda2d..604d0fc 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -246,6 +246,12 @@ async function postEvent (topic, payload) { payload } await busApiClient.postEvent(message) + + // Post to the aggregate topic + message.payload.originalTopic = topic + message.topic = config.UBAHN_AGGREGATE_TOPIC + logger.debug(`Posting event to aggregate topic ${message.topic}`) + await busApiClient.postEvent(message) } module.exports = {