feat: Add migration script for phone number normalisation

CHANGELOG.md

@@ -7,7 +7,27 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [7.0.12] - 2023-11-16
 
-- Adds Phone Number normalisation
+In this release, the core API routes have been updated to incorporate phone number normalization before processing. Consequently, existing entries in the database also need to undergo normalization. To facilitate this, we have included a migration script to normalize phone numbers for all the existing entries.
+
+**NOTE**: You can skip the migration if you are not using passwordless via phone number.
+
+### Migration steps
+
+This script updates the `phone_number` column in the `passwordless_users`, `passwordless_user_to_tenant`, and `passwordless_devices` tables with their respective normalized values. This script is idempotent and can be run multiple times without any issue. Follow the steps below to run the script: 
+
+1. Ensure that the core is already upgraded to version 7.0.12 (CDI version 4.0)
+2. Run the migration script
+
+    Make sure your Node.js version is 16 or above to run the script. Locate the migration script at `supertokens-core/migration_scripts/to_version_7_0_12/index.js`. Modify the script by updating the `DB_HOST`, `DB_USER`, `DB_PASSWORD`, and `DB_NAME` variables with the correct values. Subsequently, run the following commands to initiate the script:
+
+    ```bash
+       $ git clone https://github.com/supertokens/supertokens-core.git
+       $ cd supertokens-core/migration_scripts/to_version_7_0_12
+       $ npm install
+       $ npm start
+    ```
+
+    Performance Note: On average, the script takes 19s for every 1000 rows with a maximum of 1 connection, 4.7s with a maximum of 5 connections (default), and 4.5s with a maximum of 10 connections. Increasing the `MAX_POOL_SIZE` allows the script to leverage more connections simultaneously, potentially improving execution speed.
 
 ## [7.0.11] - 2023-11-10
 

migration_scripts/to_version_7_0_12/.gitignore

@@ -0,0 +1 @@
+node_modules

migration_scripts/to_version_7_0_12/index.js

@@ -0,0 +1,128 @@
+/*
+ *    Copyright (c) 2023, VRAI Labs and/or its affiliates. All rights reserved.
+ *
+ *    This software is licensed under the Apache License, Version 2.0 (the
+ *    "License") as published by the Apache Software Foundation.
+ *
+ *    You may not use this file except in compliance with the License. You may
+ *    obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ */
+
+const libphonenumber = require('libphonenumber-js/max');
+
+// Update the following credentials before running the script
+const DB_HOST = "";
+const DB_USER = "";
+const DB_PASSWORD = "";
+const DB_NAME = "";
+const CLIENT = ""; // Use "pg" for PostgreSQL and "mysql2" for MySQL DB
+
+const MIN_POOL_SIZE = 0;
+const MAX_POOL_SIZE = 5;
+const QUERY_TIMEOUT = 60000;
+
+if (!DB_HOST || !CLIENT) {
+  console.error('Please update the DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE and CLIENT variables before running the script.');
+  return;
+}
+
+const knex = require('knex')({
+  client: CLIENT,
+  connection: {
+    host: DB_HOST,
+    user: DB_USER,
+    password: DB_PASSWORD,
+    database: DB_NAME,
+  },
+  pool: { min: MIN_POOL_SIZE, max: MAX_POOL_SIZE }
+});
+
+function getUpdatePromise(table, entry, normalizedPhoneNumber) {
+  if (table === 'passwordless_devices') {
+    return knex.raw(`UPDATE ${table} SET phone_number = ? WHERE app_id = ? AND tenant_id = ? AND device_id_hash = ?`, [normalizedPhoneNumber, entry.app_id, entry.tenant_id, entry.device_id_hash]).timeout(QUERY_TIMEOUT, { cancel: true });
+  } else if (table === 'passwordless_users') {
+    // Since passwordless_users and passwordless_user_to_tenant are consistent. We can update both tables at the same time. For consistency, we will use a transaction.
+    return knex.transaction(async trx => {
+      await trx.raw(`UPDATE passwordless_users SET phone_number = ? WHERE app_id = ? AND user_id = ?`, [normalizedPhoneNumber, entry.app_id, entry.user_id]).timeout(QUERY_TIMEOUT, { cancel: true });
+      await trx.raw(`UPDATE passwordless_user_to_tenant SET phone_number = ? WHERE app_id = ? AND user_id = ?`, [normalizedPhoneNumber, entry.app_id, entry.user_id]).timeout(QUERY_TIMEOUT, { cancel: true });
+    });
+  } else {
+    throw new Error(`Invalid table name: ${table}`);
+  }
+}
+
+function getNormalizedPhoneNumber(phoneNumber) {
+  try {
+    return libphonenumber.parsePhoneNumber(phoneNumber, { extract: false }).format('E.164');
+  } catch (error) {
+    return null;
+  }
+}
+
+async function updatePhoneNumbers(table) {
+  const batchSize = 1000;
+  let offset = 0;
+  let totalUpdatedRows = 0;
+
+  try {
+    let totalRows = await knex.raw(`SELECT COUNT(*) as count FROM ${table} WHERE phone_number is NOT NULL`);
+    totalRows = totalRows.rows ? totalRows.rows[0].count : totalRows[0][0].count;
+
+    while (true) {
+      const entries = await knex.raw(`SELECT * FROM ${table} WHERE phone_number is NOT NULL LIMIT ${batchSize} OFFSET ${offset}`);
+      // In PostgreSQL, all rows are returned in `entries.rows`, whereas in MySQL, they can be found in `entries[0]`.
+      const rows = entries.rows ? entries.rows : entries[0];
+
+      const batchUpdates = [];
+
+      for (const entry of rows) {
+        const currentPhoneNumber = entry.phone_number;
+        const normalizedPhoneNumber = getNormalizedPhoneNumber(currentPhoneNumber);
+
+        if (normalizedPhoneNumber && normalizedPhoneNumber !== currentPhoneNumber) {
+          const updatePromise = getUpdatePromise(table, entry, normalizedPhoneNumber);
+          batchUpdates.push(updatePromise);
+        }
+      }
+
+      await Promise.all(batchUpdates);
+
+      offset += rows.length;
+      totalUpdatedRows += batchUpdates.length;
+
+      console.log(`Processed ${offset} out of ${totalRows} rows in table ${table}; ${totalUpdatedRows} rows updated`);
+
+      if (rows.length < batchSize) {
+        break;
+      }
+    }
+  } catch (error) {
+    console.error(`Error normalising phone numbers for table ${table}: Retry running the script and if the error persists after retrying then create an issue at https://github.com/supertokens/supertokens-core/issues`);
+    throw error;
+  }
+}
+
+async function runScript() {
+  const tables = ['passwordless_users', 'passwordless_devices'];
+
+  try {
+    for (const table of tables) {
+      await updatePhoneNumbers(table);
+      console.log(`\n\n\n`);
+    }
+    console.log('Finished normalising phone numbers!');
+  } catch (error) {
+    console.error(error);
+  } finally {
+    knex.destroy();
+  }
+
+}
+
+runScript();