Async rehydrate with index! (#542)

* Add fromFS option to /ipfs path

* Add migration to support direct dir cid queries and make rehydration async

* Add comment

* Fix lint

* Remove commented out code

* Fix comma

* Remove \n\n\n

* Clean up

* Fix index

* Rebase & install

* Lint

* Clean up and paginate query

* Fix migration

* Comment out update

* Reset route changes

* Lint fix

creator-node/sequelize/migrations/20200623002237-add-parent-file-data.js

@@ -0,0 +1,73 @@
+'use strict'
+
+/**
+ * Adds two columns to the `files` table
+ * - fileName: the actual queryable source filename stripped of any prefixes
+ * - dirMultihash: if the file is in an IPFS directory, the CID/multihash for the parent dir
+ */
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    const transaction = await queryInterface.sequelize.transaction()
+    try {
+      await queryInterface.addColumn(
+        'Files',
+        'fileName',
+        {
+          type: Sequelize.TEXT,
+          allowNull: true
+        },
+        { transaction }
+      )
+      await queryInterface.addColumn(
+        'Files',
+        'dirMultihash',
+        {
+          type: Sequelize.TEXT,
+          allowNull: true
+        },
+        { transaction }
+      )
+      await queryInterface.addIndex('Files', ['dirMultihash'], { transaction })
+
+      // For reference, this is what the values of the columns should be
+      // inherited from sourceFile & storagePath
+      // This bulk update is run separaely.
+
+      // await queryInterface.sequelize.query(`
+      //   UPDATE "Files" SET
+      //   "fileName" = regexp_replace("sourceFile", '(.*)\/','','g'),
+      //   "dirMultihash" = CASE
+      //     WHEN "type" = 'image' THEN regexp_replace("storagePath", '^\/.*/(Qm.*)\/(Qm.*)$','\\1','g')
+      //     ELSE null
+      //   END
+      // `, { transaction })
+
+      await transaction.commit()
+    } catch (err) {
+      await transaction.rollback()
+      throw err
+    }
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    const transaction = await queryInterface.sequelize.transaction()
+    try {
+      await queryInterface.removeColumn(
+        'Files',
+        'fileName',
+        { transaction }
+      )
+      await queryInterface.removeColumn(
+        'Files',
+        'dirMultihash',
+        { transaction }
+      )
+
+      await queryInterface.removeIndex('Files', ['dirMultihash'], { transaction })
+      await transaction.commit()
+    } catch (err) {
+      await transaction.rollback()
+      throw err
+    }
+  }
+}

creator-node/src/index.js

@@ -13,8 +13,8 @@ const { runMigrations } = require('./migrationManager')
 const { logger } = require('./logging')
 const BlacklistManager = require('./blacklistManager')
 
-const exitWithError = (msg) => {
-  logger.error(msg)
+const exitWithError = (...msg) => {
+  logger.error(...msg)
   process.exit(1)
 }
 

creator-node/src/models/file.js

@@ -27,6 +27,17 @@ module.exports = (sequelize, DataTypes) => {
       type: DataTypes.TEXT,
       allowNull: true
     },
+    // the queryable filename if this file entry is inside a dir
+    // used for images that are stored in IPFS directories (e.g. /CID/fileName.jpg)
+    fileName: {
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
+    // the parent dir's CID if this file entry is inside a dir
+    dirMultihash: {
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
     storagePath: { // format: '/file_storage/__multihash__
       type: DataTypes.TEXT,
       allowNull: false
@@ -38,7 +49,19 @@ module.exports = (sequelize, DataTypes) => {
         isIn: [['track', 'metadata', 'image', 'dir', 'copy320']]
       }
     }
-  }, {})
+  }, {
+    indexes: [
+      {
+        fields: ['cnodeUserUUID']
+      },
+      {
+        fields: ['multihash']
+      },
+      {
+        fields: ['dirMultihash']
+      }
+    ]
+  })
 
   File.associate = function (models) {
     File.belongsTo(models.CNodeUser, {