[CON-506] Migrate files with custom storage paths (#4404)

creator-node/src/config.js

@@ -41,11 +41,17 @@ const config = convict({
     default: '/file_storage'
   },
   migrateFilesWithLegacyStoragePath: {
-    doc: 'True to copy files with a legacy storage to the new storage path',
+    doc: 'True to copy files with a legacy storage path to the new storage path specified by the "storagePath" config option',
     format: Boolean,
     env: 'migrateFilesWithLegacyStoragePath',
     default: true
   },
+  migrateFilesWithCustomStoragePath: {
+    doc: 'True to copy files with a non-standard storage path to the new storage path specified by the "storagePath" config option',
+    format: Boolean,
+    env: 'migrateFilesWithCustomStoragePath',
+    default: true
+  },
   redisHost: {
     doc: 'Redis host name',
     format: String,

creator-node/src/dbManager.js

@@ -3,6 +3,7 @@ const { isEmpty } = require('lodash')
 
 const { logger } = require('./logging')
 const models = require('./models')
+const config = require('./config')
 const sequelize = models.sequelize
 const { QueryTypes } = models.Sequelize
 
@@ -446,7 +447,7 @@ class DBManager {
           [sequelize.Op.like]: '/file_storage/%'
         }
       },
-      order: [['multihash', 'ASC']],
+      order: [['multihash', 'DESC']],
       limit: batchSize
     })
     logger.debug(
@@ -468,6 +469,35 @@ class DBManager {
     return DBManager._getLegacyStoragePathsAndCids(cursor, batchSize, true)
   }
 
+  static async getCustomStoragePathsRecords(cursor, batchSize) {
+    const queryResult = await models.File.findAll({
+      attributes: [
+        'storagePath',
+        'multihash',
+        'dirMultihash',
+        'fileName',
+        'trackBlockchainId',
+        'fileUUID'
+      ],
+      where: {
+        multihash: { [sequelize.Op.gte]: cursor },
+        storagePath: {
+          [sequelize.Op.notLike]: `${config.get('storagePath')}/%`,
+          [sequelize.Op.notLike]: '/file_storage/%'
+        }
+      },
+      order: [['multihash', 'DESC']],
+      limit: batchSize
+    })
+    logger.debug(
+      `queryResult for custom storagePaths with cursor ${cursor}: ${JSON.stringify(
+        queryResult || {}
+      )}`
+    )
+    if (isEmpty(queryResult)) return []
+    return queryResult
+  }
+
   static async updateLegacyPathDbRows(copiedFilePaths, logger) {
     if (!copiedFilePaths?.length) return true
     const transaction = await models.sequelize.transaction()