From 8b5820fac20a7f040ce29af5c2cee855d58d9301 Mon Sep 17 00:00:00 2001 From: Aranyajit Date: Wed, 31 Jan 2024 14:00:52 +0530 Subject: [PATCH 1/3] enhanced script --- scripts/process-remaining-ubahn-uuids.js | 128 +++++++++++++++++------ 1 file changed, 95 insertions(+), 33 deletions(-) diff --git a/scripts/process-remaining-ubahn-uuids.js b/scripts/process-remaining-ubahn-uuids.js index ea7de5a..e32646e 100644 --- a/scripts/process-remaining-ubahn-uuids.js +++ b/scripts/process-remaining-ubahn-uuids.js @@ -3,61 +3,123 @@ const tcUserId = require('./common/tcUserId') const Sequelize = require('sequelize') const _ = require('lodash') const helper = require('../src/common/helper') +const request = require('superagent') +const config = require('config') -// read cli arguments, pass the table name -const tableName = process.argv[2] -// read cli arguments, pass the column names to update -const columnNames = process.argv[3] +/** + * + * @param {string} handle - The member handle + * @returns {Promise} The member details from the member API + */ +async function getMemberDetailsByHandle(handle) { + const token = await helper.getM2MToken() + let res + try { + res = await request + .get(`${config.TOPCODER_MEMBERS_API}/${handle}`) + .query({ + fields: 'userId,handle,handleLower,firstName,lastName,email' + }) + .set('Authorization', `Bearer ${token}`) + .set('Accept', 'application/json') + return res.body + } catch (error) { + console.log(`Unable to find member with handle ${handle}`) + return {} + } +} const processRemainingUUIDs = async (tableName, columnNames) => { const dbUrl = process.env.UBAHN_DB_URL const MODE = process.env.MODE || 'test' + const handleToIDMap = {} if (_.isUndefined(dbUrl) || _.isNull(dbUrl)) { console.log('Ubahn DB URL not set, exiting!') process.exit(0) } + // only for readability + console.log() + console.log('---------------------------------------------------------------------------------------------------------------') + console.log() + for (const columnName of _.split(columnNames, ',')) { - const query = `SELECT DISTINCT ${columnName} FROM bookings.${tableName} WHERE LENGTH(${columnName}) > 9 AND ${columnName} <> '00000000-0000-0000-0000-000000000000';` - let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT }) - - if (results.length > 0) { - results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`])) - console.log(`result: ${JSON.stringify(results)}`) - - const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl) - const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results) - - const handleToIDMap = {} - const batches = _.chunk(Object.values(uuidToHandleMap), 30) - for (const batch of batches) { - const memberAPIRes = await helper.getMemberDetailsByHandles(batch) - _.forEach(memberAPIRes, member => { - handleToIDMap[member.handleLower] = member.userId - }) - } - let sql = '' - for (const [key, value] of Object.entries(uuidToHandleMap)) { - if (!_.isUndefined(handleToIDMap[value.toLowerCase()])) { - sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';` + let transaction = await models.sequelize.transaction() + + try { + // check each column to find distinct existing uuids which have not yet been converted to TC legacy user id + const query = `SELECT DISTINCT ${columnName} FROM bookings.${tableName} WHERE LENGTH(${columnName}) > 9 AND ${columnName} <> '00000000-0000-0000-0000-000000000000';` + console.log(`Executing query in table ${tableName} against column ${columnName}`) + console.log(`SQL query: ${query}`) + let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT }) + + if (results.length > 0) { + results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`])) + console.log(`SQL query result: ${JSON.stringify(results)}`) + + // get the ubahn uuid to handle map + const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl) + const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results) + + // get the handle to legacy topcoder id map + for (const handle of Object.values(uuidToHandleMap)) { + console.log(`handle to search for ${handle}`) + if (_.isUndefined(handleToIDMap[handle])) { + const member = await getMemberDetailsByHandle(handle) + handleToIDMap[member.handleLower] = member.userId + } } + + // build the update queries + let sql = '' + for (const [key, value] of Object.entries(uuidToHandleMap)) { + if (!_.isUndefined(handleToIDMap[value.toLowerCase()])) { + sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';` + } + } + + // execute update queries if and only if it's not in test mode + if (sql !== '') { + console.log(`SQL UPDATE statements: ${sql}`) + if (MODE !== 'test') { + console.log(`Executing UPDATE statements`) + await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE, transaction: transaction }) + } + } else { + console.log(`No UPDATE statements to execute against column ${columnName}`) + } + } else { + console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`) } - console.log(`UPDATE statements: ${sql}`) - if (MODE !== test) { - await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE }) - } - } else { - console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`) + + // only for readability + console.log('---------------------------------------------------------------------------------------------------------------') + console.log() + + await transaction.commit() + + } catch (error) { + console.log(`Error encountered`) + console.error(JSON.stringify(error)) + await transaction.rollback() } } + + console.log(`DONE processing table ${tableName}`) } +// read cli arguments, pass the table name +const tableName = process.argv[2] +// read cli arguments, pass the column names to update +const columnNames = process.argv[3] + processRemainingUUIDs(tableName, columnNames).then(res => { console.log(`Processed remaining records for model '${tableName}' against columns: ${columnNames}`) process.exit(0) }).catch(err => { + console.log(`Error encountered!`) console.error(`${JSON.stringify(err)}`) process.exit(1) -}) +}) \ No newline at end of file From 62a1f454571ae43ae73376cb0b45316263260dbe Mon Sep 17 00:00:00 2001 From: Kiril Kartunov Date: Wed, 31 Jan 2024 10:34:27 +0200 Subject: [PATCH 2/3] lint fixes --- scripts/process-remaining-ubahn-uuids.js | 30 +++++++++++------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/scripts/process-remaining-ubahn-uuids.js b/scripts/process-remaining-ubahn-uuids.js index e32646e..f06b5e3 100644 --- a/scripts/process-remaining-ubahn-uuids.js +++ b/scripts/process-remaining-ubahn-uuids.js @@ -7,11 +7,11 @@ const request = require('superagent') const config = require('config') /** - * + * * @param {string} handle - The member handle - * @returns {Promise} The member details from the member API + * @returns {Promise} The member details from the member API */ -async function getMemberDetailsByHandle(handle) { +async function getMemberDetailsByHandle (handle) { const token = await helper.getM2MToken() let res try { @@ -45,8 +45,7 @@ const processRemainingUUIDs = async (tableName, columnNames) => { console.log() for (const columnName of _.split(columnNames, ',')) { - - let transaction = await models.sequelize.transaction() + const transaction = await models.sequelize.transaction() try { // check each column to find distinct existing uuids which have not yet been converted to TC legacy user id @@ -54,15 +53,15 @@ const processRemainingUUIDs = async (tableName, columnNames) => { console.log(`Executing query in table ${tableName} against column ${columnName}`) console.log(`SQL query: ${query}`) let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT }) - + if (results.length > 0) { results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`])) console.log(`SQL query result: ${JSON.stringify(results)}`) - + // get the ubahn uuid to handle map const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl) const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results) - + // get the handle to legacy topcoder id map for (const handle of Object.values(uuidToHandleMap)) { console.log(`handle to search for ${handle}`) @@ -71,7 +70,7 @@ const processRemainingUUIDs = async (tableName, columnNames) => { handleToIDMap[member.handleLower] = member.userId } } - + // build the update queries let sql = '' for (const [key, value] of Object.entries(uuidToHandleMap)) { @@ -79,12 +78,12 @@ const processRemainingUUIDs = async (tableName, columnNames) => { sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';` } } - + // execute update queries if and only if it's not in test mode if (sql !== '') { console.log(`SQL UPDATE statements: ${sql}`) if (MODE !== 'test') { - console.log(`Executing UPDATE statements`) + console.log('Executing UPDATE statements') await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE, transaction: transaction }) } } else { @@ -93,15 +92,14 @@ const processRemainingUUIDs = async (tableName, columnNames) => { } else { console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`) } - + // only for readability console.log('---------------------------------------------------------------------------------------------------------------') console.log() await transaction.commit() - } catch (error) { - console.log(`Error encountered`) + console.log('Error encountered') console.error(JSON.stringify(error)) await transaction.rollback() } @@ -119,7 +117,7 @@ processRemainingUUIDs(tableName, columnNames).then(res => { console.log(`Processed remaining records for model '${tableName}' against columns: ${columnNames}`) process.exit(0) }).catch(err => { - console.log(`Error encountered!`) + console.log('Error encountered!') console.error(`${JSON.stringify(err)}`) process.exit(1) -}) \ No newline at end of file +}) From 6a7c92460923e462e15b2ccd5458110d77a60ea9 Mon Sep 17 00:00:00 2001 From: Aranyajit Date: Wed, 31 Jan 2024 16:25:06 +0530 Subject: [PATCH 3/3] enhance scripts and remove duplicate sql statements --- scripts/alter-bookings-db-tcUserId.sql | 4 ---- scripts/process-remaining-ubahn-uuids.js | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/alter-bookings-db-tcUserId.sql b/scripts/alter-bookings-db-tcUserId.sql index 54bd638..5f0fd9d 100644 --- a/scripts/alter-bookings-db-tcUserId.sql +++ b/scripts/alter-bookings-db-tcUserId.sql @@ -28,10 +28,6 @@ ALTER COLUMN member_id TYPE TEXT, ALTER COLUMN created_by TYPE TEXT, ALTER COLUMN updated_by TYPE TEXT; -ALTER TABLE bookings.roles -ALTER COLUMN created_by TYPE TEXT, -ALTER COLUMN updated_by TYPE TEXT; - ALTER TABLE bookings.user_meeting_settings ALTER COLUMN created_by TYPE TEXT, ALTER COLUMN updated_by TYPE TEXT; diff --git a/scripts/process-remaining-ubahn-uuids.js b/scripts/process-remaining-ubahn-uuids.js index f06b5e3..c29eea4 100644 --- a/scripts/process-remaining-ubahn-uuids.js +++ b/scripts/process-remaining-ubahn-uuids.js @@ -18,7 +18,7 @@ async function getMemberDetailsByHandle (handle) { res = await request .get(`${config.TOPCODER_MEMBERS_API}/${handle}`) .query({ - fields: 'userId,handle,handleLower,firstName,lastName,email' + fields: 'userId,handle,handleLower' }) .set('Authorization', `Bearer ${token}`) .set('Accept', 'application/json')