User/Track update ipfs bug fix

creator-node/package.json

@@ -51,7 +51,8 @@
     "sequelize": "^4.41.2",
     "shortid": "^2.2.14",
     "umzug": "^2.2.0",
-    "uuid": "3.3.2"
+    "uuid": "3.3.2",
+    "fs-extra": "^9.0.1"
   },
   "devDependencies": {
     "mocha": "^5.2.0",
@@ -61,7 +62,6 @@
     "sequelize-cli": "^5.3.0",
     "sinon": "^7.0.0",
     "standard": "^12.0.1",
-    "fs-extra": "^9.0.1",
     "supertest": "^3.3.0",
     "proxyquire": "^2.1.3"
   },

creator-node/src/fileManager.js

@@ -23,8 +23,6 @@ const AUDIO_MIME_TYPE_REGEX = /audio\/(.*)/
 
 /**
  * Adds file to IPFS then saves file to disk under /multihash name
- *
-
  */
 async function saveFileFromBufferToIPFSAndDisk (req, buffer) {
   // make sure user has authenticated before saving file

creator-node/src/resizeImage.js

@@ -2,11 +2,8 @@ const Jimp = require('jimp')
 const ExifParser = require('exif-parser')
 const { logger: genericLogger } = require('./logging')
 const { ipfs } = require('./ipfsClient')
-const fs = require('fs')
+const fs = require('fs-extra')
 const path = require('path')
-const { promisify } = require('util')
-const writeFile = promisify(fs.writeFile)
-const mkdir = promisify(fs.mkdir)
 
 const MAX_HEIGHT = 6000 // No image should be taller than this.
 const COLOR_WHITE = 0xFFFFFFFF
@@ -158,29 +155,29 @@ module.exports = async (job) => {
     files: []
   }
 
+  // Create dir on disk
+  await fs.ensureDir(dirDestPath)
+
+  // Save all image file buffers to disk
   try {
-    await mkdir(dirDestPath)
+    // Slice ipfsAddResp to remove dir entry at last index
+    const ipfsFileResps = ipfsAddResp.slice(0, ipfsAddResp.length - 1)
+
+    await Promise.all(ipfsFileResps.map(async (fileResp, i) => {
+      // Save file to disk
+      const destPath = path.join(storagePath, dirCID, fileResp.hash)
+      await fs.writeFile(destPath, resizes[i])
+
+      // Append saved file info to response object
+      resp.files.push({
+        multihash: fileResp.hash,
+        sourceFile: fileResp.path,
+        storagePath: destPath
+      })
+    }))
   } catch (e) {
-    // if error = 'already exists', ignore else throw
-    if (e.message.indexOf('already exists') < 0) throw e
+    throw new Error(`Failed to write files to disk after resizing ${e}`)
   }
 
-  const ipfsFileResps = ipfsAddResp.slice(0, ipfsAddResp.length - 1)
-  await Promise.all(ipfsFileResps.map(async (fileResp, i) => {
-    logger.info('file CID', fileResp.hash)
-
-    // Save file to disk
-    const destPath = path.join(storagePath, dirCID, fileResp.hash)
-    await writeFile(destPath, resizes[i])
-
-    logger.info('Added file', fileResp, file)
-
-    resp.files.push({
-      multihash: fileResp.hash,
-      sourceFile: fileResp.path,
-      storagePath: destPath
-    })
-  }))
-
   return Promise.resolve(resp)
 }

creator-node/src/routes/audiusUsers.js

@@ -4,7 +4,7 @@ const fs = require('fs')
 const models = require('../models')
 const { saveFileFromBufferToIPFSAndDisk } = require('../fileManager')
 const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers')
-const { getFileUUIDForImageCID } = require('../utils')
+const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils')
 const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares')
 const DBManager = require('../dbManager')
 
@@ -85,8 +85,10 @@ module.exports = function (app) {
     // Get coverArtFileUUID and profilePicFileUUID for multihashes in metadata object, if present.
     let coverArtFileUUID, profilePicFileUUID
     try {
-      coverArtFileUUID = await getFileUUIDForImageCID(req, metadataJSON.cover_photo_sizes)
-      profilePicFileUUID = await getFileUUIDForImageCID(req, metadataJSON.profile_picture_sizes)
+      [coverArtFileUUID, profilePicFileUUID] = await Promise.all([
+        validateStateForImageDirCIDAndReturnFileUUID(req, metadataJSON.cover_photo_sizes),
+        validateStateForImageDirCIDAndReturnFileUUID(req, metadataJSON.profile_picture_sizes)
+      ])
     } catch (e) {
       return errorResponseBadRequest(e.message)
     }

creator-node/src/routes/files.js

@@ -1,5 +1,5 @@
 const Redis = require('ioredis')
-const fs = require('fs')
+const fs = require('fs-extra')
 const path = require('path')
 var contentDisposition = require('content-disposition')
 
@@ -353,6 +353,54 @@ module.exports = function (app) {
       return errorResponseServerError(e)
     }
 
+    /**
+     * Ensure image files written to disk match dirCID returned from resizeImage
+     */
+
+    const ipfs = req.app.get('ipfsLatestAPI')
+
+    const dirCID = resizeResp.dir.dirCID
+
+    // build ipfs add array
+    let ipfsAddArray = []
+    try {
+      await Promise.all(resizeResp.files.map(async function (file) {
+        const fileBuffer = await fs.readFile(file.storagePath)
+        ipfsAddArray.push({
+          path: file.sourceFile,
+          content: fileBuffer
+        })
+      }))
+    } catch (e) {
+      throw new Error(`Failed to build ipfs add array for dirCID ${dirCID} ${e}`)
+    }
+
+    // Re-compute dirCID from all image files to ensure it matches dirCID returned above
+    let ipfsAddRespArr
+    try {
+      const ipfsAddResp = await ipfs.add(
+        ipfsAddArray,
+        {
+          pin: false,
+          onlyHash: true,
+          timeout: 1000
+        }
+      )
+      ipfsAddRespArr = []
+      for await (const resp of ipfsAddResp) {
+        ipfsAddRespArr.push(resp)
+      }
+    } catch (e) {
+      // If ipfs.add op fails, log error and move on, since this is an ipfs error and not an image upload error
+      req.logger.info(`Error calling ipfs.add on dir to re-compute dirCID ${dirCID} ${e}`)
+    }
+
+    // Ensure actual and expected dirCIDs match
+    const expectedDirCID = ipfsAddRespArr[ipfsAddRespArr.length - 1].cid.toString()
+    if (expectedDirCID !== dirCID) {
+      throw new Error(`Image file validation failed - dirCIDs do not match for dirCID ${dirCID}`)
+    }
+
     // Record image file entries in DB
     const transaction = await models.sequelize.transaction()
     try {
@@ -373,20 +421,21 @@ module.exports = function (app) {
           sourceFile: file.sourceFile,
           storagePath: file.storagePath,
           type: 'image', // TODO - replace with models enum
-          dirMultihash: resizeResp.dir.dirCID,
+          dirMultihash: dirCID,
           fileName: file.sourceFile.split('/').slice(-1)[0]
         }
         await DBManager.createNewDataRecord(createImageFileQueryObj, cnodeUserUUID, models.File, transaction)
       }
 
       req.logger.info(`route time = ${Date.now() - routestart}`)
       await transaction.commit()
-      triggerSecondarySyncs(req)
-      return successResponse({ dirCID: resizeResp.dir.dirCID })
     } catch (e) {
       await transaction.rollback()
       return errorResponseServerError(e)
     }
+
+    triggerSecondarySyncs(req)
+    return successResponse({ dirCID })
   }))
 
   app.get('/ipfs_peer_info', handleResponse(async (req, res) => {

creator-node/src/routes/nodeSync.js

@@ -285,7 +285,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync
           // Spread + set uniq's the array
           userReplicaSet = [...new Set(userReplicaSet)]
         } catch (e) {
-          req.logger.error(redisKey, `Couldn't get user's replica sets, can't use cnode gateways in saveFileForMultihash`)
+          req.logger.error(redisKey, `Couldn't get user's replica sets, can't use cnode gateways in saveFileForMultihash - ${e.message}`)
         }
       }
 

creator-node/src/routes/tracks.js

@@ -14,7 +14,7 @@ const {
   errorResponseServerError,
   errorResponseForbidden
 } = require('../apiHelpers')
-const { getFileUUIDForImageCID } = require('../utils')
+const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils')
 const { authMiddleware, ensurePrimaryMiddleware, syncLockMiddleware, triggerSecondarySyncs } = require('../middlewares')
 const TranscodingQueue = require('../TranscodingQueue')
 const { getCID } = require('./files')
@@ -310,7 +310,7 @@ module.exports = function (app) {
     // Get coverArtFileUUID for multihash in metadata object, else error
     let coverArtFileUUID
     try {
-      coverArtFileUUID = await getFileUUIDForImageCID(req, metadataJSON.cover_art_sizes)
+      coverArtFileUUID = await validateStateForImageDirCIDAndReturnFileUUID(req, metadataJSON.cover_art_sizes)
     } catch (e) {
       return errorResponseServerError(e.message)
     }

creator-node/src/utils.js

@@ -1,8 +1,6 @@
 const { recoverPersonalSignature } = require('eth-sig-util')
-const { promisify } = require('util')
-const fs = require('fs')
+const fs = require('fs-extra')
 const path = require('path')
-const mkdir = promisify(fs.mkdir)
 const { BufferListStream } = require('bl')
 const axios = require('axios')
 
@@ -26,47 +24,46 @@ class Utils {
   }
 }
 
-async function getFileUUIDForImageCID (req, imageCID) {
-  const ipfs = req.app.get('ipfsAPI')
-  if (imageCID) { // assumes imageCIDs are optional params
-    // Ensure CID points to a dir, not file
-    let cidIsFile = false
-    try {
-      await ipfs.cat(imageCID, { length: 1 })
-      cidIsFile = true
-    } catch (e) {
-      // Ensure file exists for dirCID
-      const dirFile = await models.File.findOne({
-        where: { multihash: imageCID, cnodeUserUUID: req.session.cnodeUserUUID, type: 'dir' }
-      })
-      if (!dirFile) {
-        throw new Error(`No file stored in DB for dir CID ${imageCID}`)
-      }
+/**
+ * Ensure DB and disk records exist for dirCID and its contents
+ * Return fileUUID for dir DB record
+ * This function does not do further validation since image_upload provides remaining guarantees
+ */
+async function validateStateForImageDirCIDAndReturnFileUUID (req, imageDirCID) {
+  if (!imageDirCID) {
+    return null
+  }
+  req.logger.info(`Beginning validateStateForImageDirCIDAndReturnFileUUID for imageDirCID ${imageDirCID}`)
 
-      // Ensure file refs exist in DB for every file in dir
-      const dirContents = await ipfs.ls(imageCID)
-      req.logger.info(dirContents)
+  // Ensure file exists for dirCID
+  const dirFile = await models.File.findOne({
+    where: { multihash: imageDirCID, cnodeUserUUID: req.session.cnodeUserUUID, type: 'dir' }
+  })
+  if (!dirFile) {
+    throw new Error(`No file stored in DB for imageDirCID ${imageDirCID}`)
+  }
 
-      // Iterates through directory contents but returns upon first iteration
-      // TODO: refactor to remove for-loop
-      for (let fileObj of dirContents) {
-        if (!fileObj.hasOwnProperty('hash') || !fileObj.hash) {
-          throw new Error(`Malformatted dir contents for dirCID ${imageCID}. Cannot process.`)
-        }
+  // Ensure dir exists on disk
+  if (!(await fs.pathExists(dirFile.storagePath))) {
+    throw new Error(`No dir found on disk for imageDirCID ${imageDirCID} at expected path ${dirFile.storagePath}`)
+  }
 
-        const imageFile = await models.File.findOne({
-          where: { multihash: fileObj.hash, cnodeUserUUID: req.session.cnodeUserUUID, type: 'image' }
-        })
-        if (!imageFile) {
-          throw new Error(`No file ref stored in DB for CID ${fileObj.hash} in dirCID ${imageCID}`)
-        }
-        return dirFile.fileUUID
-      }
-    }
-    if (cidIsFile) {
-      throw new Error(`CID ${imageCID} must point to a valid directory on IPFS`)
+  const imageFiles = await models.File.findAll({
+    where: { dirMultihash: imageDirCID, cnodeUserUUID: req.session.cnodeUserUUID, type: 'image' }
+  })
+  if (!imageFiles) {
+    throw new Error(`No image file records found in DB for imageDirCID ${imageDirCID}`)
+  }
+
+  // Ensure every file exists on disk
+  await Promise.all(imageFiles.map(async function (imageFile) {
+    if (!(await fs.pathExists(imageFile.storagePath))) {
+      throw new Error(`No file found on disk for imageDirCID ${imageDirCID} image file at path ${imageFile.path}`)
     }
-  } else return null
+  }))
+
+  req.logger.info(`Completed validateStateForImageDirCIDAndReturnFileUUID for imageDirCID ${imageDirCID}`)
+  return dirFile.fileUUID
 }
 
 async function getIPFSPeerId (ipfs) {
@@ -417,7 +414,7 @@ async function rehydrateIpfsDirFromFsIfNecessary (dirHash, logContext) {
 
 async function createDirForFile (fileStoragePath) {
   const dir = path.dirname(fileStoragePath)
-  await mkdir(dir, { recursive: true })
+  await fs.ensureDir(dir)
 }
 
 async function writeStreamToFileSystem (stream, expectedStoragePath, createDir = false) {
@@ -437,7 +434,7 @@ async function writeStreamToFileSystem (stream, expectedStoragePath, createDir =
 }
 
 module.exports = Utils
-module.exports.getFileUUIDForImageCID = getFileUUIDForImageCID
+module.exports.validateStateForImageDirCIDAndReturnFileUUID = validateStateForImageDirCIDAndReturnFileUUID
 module.exports.getIPFSPeerId = getIPFSPeerId
 module.exports.rehydrateIpfsFromFsIfNecessary = rehydrateIpfsFromFsIfNecessary
 module.exports.rehydrateIpfsDirFromFsIfNecessary = rehydrateIpfsDirFromFsIfNecessary