Update dev data insertion and deletion scripts to also insert and delete into elasticsearch

Author: callmekatootie

scripts/constants.js

@@ -0,0 +1,84 @@
+/**
+ * This module contains es resources configuration.
+ * Identical to the one from ES processor, but updated to work with the api
+ */
+
+const config = require('config')
+
+const topResources = {
+  achievementprovider: {
+    index: config.get('ES.DOCUMENTS.achievementprovider.index'),
+    type: config.get('ES.DOCUMENTS.achievementprovider.type')
+  },
+  attribute: {
+    index: config.get('ES.DOCUMENTS.attribute.index'),
+    type: config.get('ES.DOCUMENTS.attribute.type')
+  },
+  attributegroup: {
+    index: config.get('ES.DOCUMENTS.attributegroup.index'),
+    type: config.get('ES.DOCUMENTS.attributegroup.type')
+  },
+  organization: {
+    index: config.get('ES.DOCUMENTS.organization.index'),
+    type: config.get('ES.DOCUMENTS.organization.type')
+  },
+  role: {
+    index: config.get('ES.DOCUMENTS.role.index'),
+    type: config.get('ES.DOCUMENTS.role.type')
+  },
+  skill: {
+    index: config.get('ES.DOCUMENTS.skill.index'),
+    type: config.get('ES.DOCUMENTS.skill.type')
+  },
+  skillprovider: {
+    index: config.get('ES.DOCUMENTS.skillprovider.index'),
+    type: config.get('ES.DOCUMENTS.skillprovider.type')
+  },
+  user: {
+    index: config.get('ES.DOCUMENTS.user.index'),
+    type: config.get('ES.DOCUMENTS.user.type')
+  }
+}
+
+const userResources = {
+  achievement: {
+    propertyName: config.get('ES.DOCUMENTS.achievement.userField'),
+    relateKey: 'achievementsProviderId'
+  },
+  externalprofile: {
+    propertyName: config.get('ES.DOCUMENTS.externalprofile.userField'),
+    relateKey: 'organizationId'
+  },
+  userattribute: {
+    propertyName: config.get('ES.DOCUMENTS.userattribute.userField'),
+    relateKey: 'attributeId'
+  },
+  userrole: {
+    propertyName: config.get('ES.DOCUMENTS.userrole.userField'),
+    relateKey: 'roleId'
+  },
+  userskill: {
+    propertyName: config.get('ES.DOCUMENTS.userskill.userField'),
+    relateKey: 'skillId'
+  }
+}
+
+const modelToESIndexMapping = {
+  User: 'user',
+  Role: 'role',
+  SkillsProvider: 'skillprovider',
+  Organization: 'organization',
+  Skill: 'skill',
+  UsersRole: 'userrole',
+  UsersSkill: 'userskill',
+  Achievement: 'achievement',
+  ExternalProfile: 'externalprofile',
+  AchievementsProvider: 'achievementprovider',
+  AttributeGroup: 'attributegroup',
+  Attribute: 'attribute',
+  UserAttribute: 'userattribute'
+}
+
+module.exports = {
+  topResources, userResources, modelToESIndexMapping
+}

scripts/db/data/UsersAttribute.json renamed to scripts/db/data/UserAttribute.json

@@ -3,14 +3,24 @@
  */
 const models = require('../../src/models')
 const logger = require('../../src/common/logger')
+const { topResources, modelToESIndexMapping } = require('../constants')
+const { getESClient } = require('../../src/common/es-client')
 
 async function main () {
+  const client = getESClient()
   const keys = Object.keys(models)
   for (let i = 0; i < keys.length; i++) {
     const key = keys[i]
     if (models[key].tableName) {
+      const esResourceName = modelToESIndexMapping[key]
       try {
         await models.DBHelper.drop(models[key])
+
+        if (_.includes(_.keys(topResources), esResourceName)) {
+          await client.indices.delete({
+            index: topResources[esResourceName].index
+          })
+        }
       } catch (e) {
         console.error(e)
         logger.warn(`drop table ${key} failed`)

scripts/db/dropAll.js

@@ -1,13 +1,77 @@
+const _ = require('lodash')
 const models = require('../../src/models')
 const logger = require('../../src/common/logger')
+const { getESClient } = require('../../src/common/es-client')
+const { topResources, userResources, modelToESIndexMapping } = require('../constants')
+
+async function insertIntoES (modelName, body) {
+  const esResourceName = modelToESIndexMapping[modelName]
+
+  if (!esResourceName) {
+    logger.error(`Cannot insert data into model ${modelName}. No equivalent elasticsearch index found`)
+
+    return
+  }
+
+  const client = getESClient()
+
+  if (_.includes(_.keys(topResources), esResourceName)) {
+    await client.create({
+      index: topResources[esResourceName].index,
+      type: topResources[esResourceName].type,
+      id: body.id,
+      body,
+      refresh: 'true'
+    })
+  } else if (_.includes(_.keys(userResources), esResourceName)) {
+    const userResource = userResources[esResourceName]
+
+    const user = await client.getSource({
+      index: topResources.user.index,
+      type: topResources.user.type,
+      id: body.userId
+    })
+
+    const relateId = body[userResource.relateKey]
+
+    if (!user[userResource.propertyName]) {
+      user[userResource.propertyName] = []
+    }
+
+    if (_.some(user[userResource.propertyName], [userResource.relateKey, relateId])) {
+      logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`)
+    } else {
+      user[userResource.propertyName].push(body)
+      await client.update({
+        index: topResources.user.index,
+        type: topResources.user.type,
+        id: body.userId,
+        body: { doc: user },
+        refresh: 'true'
+      })
+    }
+  }
+}
 
 /**
  * import test data
  * @return {Promise<void>}
  */
 async function main () {
   await models.init()
-  const keys = Object.keys(models)
+
+  let keys = Object.keys(models)
+  keys = _.orderBy(keys, k => {
+    const esResourceName = modelToESIndexMapping[k]
+
+    // Create parent data first
+    if (_.includes(_.keys(topResources), esResourceName)) {
+      return -1
+    }
+
+    return 1
+  })
+
   for (let i = 0; i < keys.length; i++) {
     const key = keys[i]
     if (models[key].tableName) {
@@ -16,6 +80,7 @@ async function main () {
         await models.DBHelper.clear(models[key])
         for (let i = 0; i < data.length; i++) {
           await models.DBHelper.save(models[key], new models[key]().from(data[i]), true)
+          await insertIntoES(key, data[i])
         }
         logger.info('import data for ' + key + ' done')
       } catch (e) {