diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index 1d90a1d0dbc..6128539fecb 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -1278,8 +1278,7 @@ "at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", - "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", - "dev": true + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==" }, "atob": { "version": "2.1.2", @@ -4407,7 +4406,6 @@ "version": "9.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz", "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==", - "dev": true, "requires": { "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", @@ -4419,7 +4417,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz", "integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==", - "dev": true, "requires": { "graceful-fs": "^4.1.6", "universalify": "^1.0.0" @@ -4428,8 +4425,7 @@ "universalify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz", - "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==", - "dev": true + "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==" } } }, diff --git a/creator-node/package.json b/creator-node/package.json index 20a9488a3a2..fabad2aab46 100644 --- a/creator-node/package.json +++ b/creator-node/package.json @@ -51,7 +51,8 @@ "sequelize": "^4.41.2", "shortid": "^2.2.14", "umzug": "^2.2.0", - "uuid": "3.3.2" + "uuid": "3.3.2", + "fs-extra": "^9.0.1" }, "devDependencies": { "mocha": "^5.2.0", @@ -61,7 +62,6 @@ "sequelize-cli": "^5.3.0", "sinon": "^7.0.0", "standard": "^12.0.1", - "fs-extra": "^9.0.1", "supertest": "^3.3.0", "proxyquire": "^2.1.3" }, diff --git a/creator-node/src/fileManager.js b/creator-node/src/fileManager.js index 4f5d932741e..30538f93adc 100644 --- a/creator-node/src/fileManager.js +++ b/creator-node/src/fileManager.js @@ -23,8 +23,6 @@ const AUDIO_MIME_TYPE_REGEX = /audio\/(.*)/ /** * Adds file to IPFS then saves file to disk under /multihash name - * - */ async function saveFileFromBufferToIPFSAndDisk (req, buffer) { // make sure user has authenticated before saving file diff --git a/creator-node/src/resizeImage.js b/creator-node/src/resizeImage.js index 370cf23e332..b487f38ed07 100644 --- a/creator-node/src/resizeImage.js +++ b/creator-node/src/resizeImage.js @@ -2,11 +2,8 @@ const Jimp = require('jimp') const ExifParser = require('exif-parser') const { logger: genericLogger } = require('./logging') const { ipfs } = require('./ipfsClient') -const fs = require('fs') +const fs = require('fs-extra') const path = require('path') -const { promisify } = require('util') -const writeFile = promisify(fs.writeFile) -const mkdir = promisify(fs.mkdir) const MAX_HEIGHT = 6000 // No image should be taller than this. const COLOR_WHITE = 0xFFFFFFFF @@ -158,29 +155,29 @@ module.exports = async (job) => { files: [] } + // Create dir on disk + await fs.ensureDir(dirDestPath) + + // Save all image file buffers to disk try { - await mkdir(dirDestPath) + // Slice ipfsAddResp to remove dir entry at last index + const ipfsFileResps = ipfsAddResp.slice(0, ipfsAddResp.length - 1) + + await Promise.all(ipfsFileResps.map(async (fileResp, i) => { + // Save file to disk + const destPath = path.join(storagePath, dirCID, fileResp.hash) + await fs.writeFile(destPath, resizes[i]) + + // Append saved file info to response object + resp.files.push({ + multihash: fileResp.hash, + sourceFile: fileResp.path, + storagePath: destPath + }) + })) } catch (e) { - // if error = 'already exists', ignore else throw - if (e.message.indexOf('already exists') < 0) throw e + throw new Error(`Failed to write files to disk after resizing ${e}`) } - const ipfsFileResps = ipfsAddResp.slice(0, ipfsAddResp.length - 1) - await Promise.all(ipfsFileResps.map(async (fileResp, i) => { - logger.info('file CID', fileResp.hash) - - // Save file to disk - const destPath = path.join(storagePath, dirCID, fileResp.hash) - await writeFile(destPath, resizes[i]) - - logger.info('Added file', fileResp, file) - - resp.files.push({ - multihash: fileResp.hash, - sourceFile: fileResp.path, - storagePath: destPath - }) - })) - return Promise.resolve(resp) } diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index 4dce624d793..b9c4430f238 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -4,7 +4,7 @@ const fs = require('fs') const models = require('../models') const { saveFileFromBufferToIPFSAndDisk } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers') -const { getFileUUIDForImageCID } = require('../utils') +const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') const DBManager = require('../dbManager') @@ -85,8 +85,10 @@ module.exports = function (app) { // Get coverArtFileUUID and profilePicFileUUID for multihashes in metadata object, if present. let coverArtFileUUID, profilePicFileUUID try { - coverArtFileUUID = await getFileUUIDForImageCID(req, metadataJSON.cover_photo_sizes) - profilePicFileUUID = await getFileUUIDForImageCID(req, metadataJSON.profile_picture_sizes) + [coverArtFileUUID, profilePicFileUUID] = await Promise.all([ + validateStateForImageDirCIDAndReturnFileUUID(req, metadataJSON.cover_photo_sizes), + validateStateForImageDirCIDAndReturnFileUUID(req, metadataJSON.profile_picture_sizes) + ]) } catch (e) { return errorResponseBadRequest(e.message) } diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index c6499a563f3..76942707820 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -1,5 +1,5 @@ const Redis = require('ioredis') -const fs = require('fs') +const fs = require('fs-extra') const path = require('path') var contentDisposition = require('content-disposition') @@ -354,6 +354,54 @@ module.exports = function (app) { return errorResponseServerError(e) } + /** + * Ensure image files written to disk match dirCID returned from resizeImage + */ + + const ipfs = req.app.get('ipfsLatestAPI') + + const dirCID = resizeResp.dir.dirCID + + // build ipfs add array + let ipfsAddArray = [] + try { + await Promise.all(resizeResp.files.map(async function (file) { + const fileBuffer = await fs.readFile(file.storagePath) + ipfsAddArray.push({ + path: file.sourceFile, + content: fileBuffer + }) + })) + } catch (e) { + throw new Error(`Failed to build ipfs add array for dirCID ${dirCID} ${e}`) + } + + // Re-compute dirCID from all image files to ensure it matches dirCID returned above + let ipfsAddRespArr + try { + const ipfsAddResp = await ipfs.add( + ipfsAddArray, + { + pin: false, + onlyHash: true, + timeout: 1000 + } + ) + ipfsAddRespArr = [] + for await (const resp of ipfsAddResp) { + ipfsAddRespArr.push(resp) + } + } catch (e) { + // If ipfs.add op fails, log error and move on, since this is an ipfs error and not an image upload error + req.logger.info(`Error calling ipfs.add on dir to re-compute dirCID ${dirCID} ${e}`) + } + + // Ensure actual and expected dirCIDs match + const expectedDirCID = ipfsAddRespArr[ipfsAddRespArr.length - 1].cid.toString() + if (expectedDirCID !== dirCID) { + throw new Error(`Image file validation failed - dirCIDs do not match for dirCID ${dirCID}`) + } + // Record image file entries in DB const transaction = await models.sequelize.transaction() try { @@ -374,7 +422,7 @@ module.exports = function (app) { sourceFile: file.sourceFile, storagePath: file.storagePath, type: 'image', // TODO - replace with models enum - dirMultihash: resizeResp.dir.dirCID, + dirMultihash: dirCID, fileName: file.sourceFile.split('/').slice(-1)[0] } await DBManager.createNewDataRecord(createImageFileQueryObj, cnodeUserUUID, models.File, transaction) @@ -382,12 +430,13 @@ module.exports = function (app) { req.logger.info(`route time = ${Date.now() - routestart}`) await transaction.commit() - triggerSecondarySyncs(req) - return successResponse({ dirCID: resizeResp.dir.dirCID }) } catch (e) { await transaction.rollback() return errorResponseServerError(e) } + + triggerSecondarySyncs(req) + return successResponse({ dirCID }) })) app.get('/ipfs_peer_info', handleResponse(async (req, res) => { diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 1dac6535e13..64177a9dc65 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -285,7 +285,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync // Spread + set uniq's the array userReplicaSet = [...new Set(userReplicaSet)] } catch (e) { - req.logger.error(redisKey, `Couldn't get user's replica sets, can't use cnode gateways in saveFileForMultihash`) + req.logger.error(redisKey, `Couldn't get user's replica sets, can't use cnode gateways in saveFileForMultihash - ${e.message}`) } } diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index dda69fc7558..599f1c22cd9 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -15,7 +15,7 @@ const { errorResponseServerError, errorResponseForbidden } = require('../apiHelpers') -const { getFileUUIDForImageCID } = require('../utils') +const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils') const { authMiddleware, ensurePrimaryMiddleware, syncLockMiddleware, triggerSecondarySyncs } = require('../middlewares') const TranscodingQueue = require('../TranscodingQueue') const { getCID } = require('./files') @@ -311,7 +311,7 @@ module.exports = function (app) { // Get coverArtFileUUID for multihash in metadata object, else error let coverArtFileUUID try { - coverArtFileUUID = await getFileUUIDForImageCID(req, metadataJSON.cover_art_sizes) + coverArtFileUUID = await validateStateForImageDirCIDAndReturnFileUUID(req, metadataJSON.cover_art_sizes) } catch (e) { return errorResponseServerError(e.message) } diff --git a/creator-node/src/utils.js b/creator-node/src/utils.js index 280d781cc78..eccd25a1851 100644 --- a/creator-node/src/utils.js +++ b/creator-node/src/utils.js @@ -1,8 +1,6 @@ const { recoverPersonalSignature } = require('eth-sig-util') -const { promisify } = require('util') -const fs = require('fs') +const fs = require('fs-extra') const path = require('path') -const mkdir = promisify(fs.mkdir) const { BufferListStream } = require('bl') const axios = require('axios') @@ -26,47 +24,47 @@ class Utils { } } -async function getFileUUIDForImageCID (req, imageCID) { - const ipfs = req.app.get('ipfsAPI') - if (imageCID) { // assumes imageCIDs are optional params - // Ensure CID points to a dir, not file - let cidIsFile = false - try { - await ipfs.cat(imageCID, { length: 1 }) - cidIsFile = true - } catch (e) { - // Ensure file exists for dirCID - const dirFile = await models.File.findOne({ - where: { multihash: imageCID, cnodeUserUUID: req.session.cnodeUserUUID, type: 'dir' } - }) - if (!dirFile) { - throw new Error(`No file stored in DB for dir CID ${imageCID}`) - } +/** + * Ensure DB and disk records exist for dirCID and its contents + * Return fileUUID for dir DB record + * This function does not do further validation since image_upload provides remaining guarantees + */ +async function validateStateForImageDirCIDAndReturnFileUUID (req, imageDirCID) { + // This handles case where a user/track metadata obj contains no image CID + if (!imageDirCID) { + return null + } + req.logger.info(`Beginning validateStateForImageDirCIDAndReturnFileUUID for imageDirCID ${imageDirCID}`) - // Ensure file refs exist in DB for every file in dir - const dirContents = await ipfs.ls(imageCID) - req.logger.info(dirContents) + // Ensure file exists for dirCID + const dirFile = await models.File.findOne({ + where: { multihash: imageDirCID, cnodeUserUUID: req.session.cnodeUserUUID, type: 'dir' } + }) + if (!dirFile) { + throw new Error(`No file stored in DB for imageDirCID ${imageDirCID}`) + } - // Iterates through directory contents but returns upon first iteration - // TODO: refactor to remove for-loop - for (let fileObj of dirContents) { - if (!fileObj.hasOwnProperty('hash') || !fileObj.hash) { - throw new Error(`Malformatted dir contents for dirCID ${imageCID}. Cannot process.`) - } + // Ensure dir exists on disk + if (!(await fs.pathExists(dirFile.storagePath))) { + throw new Error(`No dir found on disk for imageDirCID ${imageDirCID} at expected path ${dirFile.storagePath}`) + } - const imageFile = await models.File.findOne({ - where: { multihash: fileObj.hash, cnodeUserUUID: req.session.cnodeUserUUID, type: 'image' } - }) - if (!imageFile) { - throw new Error(`No file ref stored in DB for CID ${fileObj.hash} in dirCID ${imageCID}`) - } - return dirFile.fileUUID - } - } - if (cidIsFile) { - throw new Error(`CID ${imageCID} must point to a valid directory on IPFS`) + const imageFiles = await models.File.findAll({ + where: { dirMultihash: imageDirCID, cnodeUserUUID: req.session.cnodeUserUUID, type: 'image' } + }) + if (!imageFiles) { + throw new Error(`No image file records found in DB for imageDirCID ${imageDirCID}`) + } + + // Ensure every file exists on disk + await Promise.all(imageFiles.map(async function (imageFile) { + if (!(await fs.pathExists(imageFile.storagePath))) { + throw new Error(`No file found on disk for imageDirCID ${imageDirCID} image file at path ${imageFile.path}`) } - } else return null + })) + + req.logger.info(`Completed validateStateForImageDirCIDAndReturnFileUUID for imageDirCID ${imageDirCID}`) + return dirFile.fileUUID } async function getIPFSPeerId (ipfs) { @@ -417,7 +415,7 @@ async function rehydrateIpfsDirFromFsIfNecessary (dirHash, logContext) { async function createDirForFile (fileStoragePath) { const dir = path.dirname(fileStoragePath) - await mkdir(dir, { recursive: true }) + await fs.ensureDir(dir) } async function writeStreamToFileSystem (stream, expectedStoragePath, createDir = false) { @@ -437,7 +435,7 @@ async function writeStreamToFileSystem (stream, expectedStoragePath, createDir = } module.exports = Utils -module.exports.getFileUUIDForImageCID = getFileUUIDForImageCID +module.exports.validateStateForImageDirCIDAndReturnFileUUID = validateStateForImageDirCIDAndReturnFileUUID module.exports.getIPFSPeerId = getIPFSPeerId module.exports.rehydrateIpfsFromFsIfNecessary = rehydrateIpfsFromFsIfNecessary module.exports.rehydrateIpfsDirFromFsIfNecessary = rehydrateIpfsDirFromFsIfNecessary diff --git a/eth-contracts/package-lock.json b/eth-contracts/package-lock.json index 6a009ce6271..99192d42958 100644 --- a/eth-contracts/package-lock.json +++ b/eth-contracts/package-lock.json @@ -40,9 +40,9 @@ } }, "@ethersproject/abstract-signer": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/@ethersproject/abstract-signer/-/abstract-signer-5.0.6.tgz", - "integrity": "sha512-h8TZBX3pL2Xx9tmsRxfWcaaI+FcJFHWvZ/vNvFjLp8zJ0kPD501LKTt2jo44LZ20N3EW68JMoyEmRQ6bpsn+iA==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/@ethersproject/abstract-signer/-/abstract-signer-5.0.7.tgz", + "integrity": "sha512-8W8gy/QutEL60EoMEpvxZ8MFAEWs/JvH5nmZ6xeLXoZvmBCasGmxqHdYjo2cxg0nevkPkq9SeenSsBBZSCx+SQ==", "requires": { "@ethersproject/abstract-provider": "^5.0.4", "@ethersproject/bignumber": "^5.0.7", @@ -124,13 +124,17 @@ } }, "@ethersproject/hash": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/@ethersproject/hash/-/hash-5.0.5.tgz", - "integrity": "sha512-GpI80/h2HDpfNKpCZoxQJCjOQloGnlD5hM1G+tZe8FQDJhEvFjJoPDuWv+NaYjJfOciKS2Axqc4Q4WamdLoUgg==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@ethersproject/hash/-/hash-5.0.6.tgz", + "integrity": "sha512-Gvh57v6BWhwnud6l7tMfQm32PRQ2DYx2WaAAQmAxAfYvmzUkpQCBstnGeNMXIL8/2wdkvcB2u+WZRWaZtsFuUQ==", "requires": { + "@ethersproject/abstract-signer": "^5.0.6", + "@ethersproject/address": "^5.0.5", + "@ethersproject/bignumber": "^5.0.8", "@ethersproject/bytes": "^5.0.4", "@ethersproject/keccak256": "^5.0.3", "@ethersproject/logger": "^5.0.5", + "@ethersproject/properties": "^5.0.4", "@ethersproject/strings": "^5.0.4" } }, @@ -227,9 +231,9 @@ } }, "@ethersproject/providers": { - "version": "5.0.12", - "resolved": "https://registry.npmjs.org/@ethersproject/providers/-/providers-5.0.12.tgz", - "integrity": "sha512-bRUEVNth+wGlm2Q0cQprVlixBWumfP9anrgAc3V2CbIh+GKvCwisVO8uRLrZOfOvTNSy6PUJi/Z4D5L+k3NAog==", + "version": "5.0.14", + "resolved": "https://registry.npmjs.org/@ethersproject/providers/-/providers-5.0.14.tgz", + "integrity": "sha512-K9QRRkkHWyprm3g4L8U9aPx5uyivznL4RYemkN2shCQumyGqFJ5SO+OtQrgebVm0JpGwFAUGugnhRUh49sjErw==", "requires": { "@ethersproject/abstract-provider": "^5.0.4", "@ethersproject/abstract-signer": "^5.0.4", @@ -374,9 +378,9 @@ } }, "@ethersproject/wallet": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/@ethersproject/wallet/-/wallet-5.0.5.tgz", - "integrity": "sha512-NbrKmsW3w+5dVOEyVCN5VAAIp3y8ckutW6AV7Lo0Hn8RO9mLT8ZFzLGp4lzgJoxkm+EV8BE+x1N6NdiOgUzRng==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/@ethersproject/wallet/-/wallet-5.0.7.tgz", + "integrity": "sha512-n2GX1+2Tc0qV8dguUcLkjNugINKvZY7u/5fEsn0skW9rz5+jHTR5IKMV6jSfXA+WjQT8UCNMvkI3CNcdhaPbTQ==", "requires": { "@ethersproject/abstract-provider": "^5.0.4", "@ethersproject/abstract-signer": "^5.0.4", @@ -4796,13 +4800,13 @@ } }, "ethers-latest": { - "version": "npm:ethers@5.0.17", - "resolved": "https://registry.npmjs.org/ethers/-/ethers-5.0.17.tgz", - "integrity": "sha512-E0MrwCttHgdD6Irfa0B9cNdX0VoWVWLusaj51+EQalkl3pqhV2zGMPncfhYbc9+4nD2u81dbX8Pk9UN5kh/jew==", + "version": "npm:ethers@5.0.19", + "resolved": "https://registry.npmjs.org/ethers/-/ethers-5.0.19.tgz", + "integrity": "sha512-0AZnUgZh98q888WAd1oI3aLeI+iyDtrupjANVtPPS7O63lVopkR/No8A1NqSkgl/rU+b2iuu2mUZor6GD4RG2w==", "requires": { "@ethersproject/abi": "5.0.7", "@ethersproject/abstract-provider": "5.0.5", - "@ethersproject/abstract-signer": "5.0.6", + "@ethersproject/abstract-signer": "5.0.7", "@ethersproject/address": "5.0.5", "@ethersproject/base64": "5.0.4", "@ethersproject/basex": "5.0.4", @@ -4810,7 +4814,7 @@ "@ethersproject/bytes": "5.0.5", "@ethersproject/constants": "5.0.5", "@ethersproject/contracts": "5.0.5", - "@ethersproject/hash": "5.0.5", + "@ethersproject/hash": "5.0.6", "@ethersproject/hdnode": "5.0.5", "@ethersproject/json-wallets": "5.0.7", "@ethersproject/keccak256": "5.0.4", @@ -4818,7 +4822,7 @@ "@ethersproject/networks": "5.0.4", "@ethersproject/pbkdf2": "5.0.4", "@ethersproject/properties": "5.0.4", - "@ethersproject/providers": "5.0.12", + "@ethersproject/providers": "5.0.14", "@ethersproject/random": "5.0.4", "@ethersproject/rlp": "5.0.4", "@ethersproject/sha2": "5.0.4", @@ -4827,7 +4831,7 @@ "@ethersproject/strings": "5.0.5", "@ethersproject/transactions": "5.0.6", "@ethersproject/units": "5.0.6", - "@ethersproject/wallet": "5.0.5", + "@ethersproject/wallet": "5.0.7", "@ethersproject/web": "5.0.9", "@ethersproject/wordlists": "5.0.5" } diff --git a/service-commands/scripts/hosts.js b/service-commands/scripts/hosts.js index 2911f22c420..e3e09f70b42 100644 --- a/service-commands/scripts/hosts.js +++ b/service-commands/scripts/hosts.js @@ -88,7 +88,7 @@ if (cmd === 'add') { throw new Error('Misconfigured local env.\nEnsure AUDIUS_REMOTE_DEV_HOST has been exported and /etc/hosts file has necessary permissions.') } const hostMappings = SERVICES.map(s => `${REMOTE_DEV_HOST} ${s}`) - hostMappings.push(`${REMOTE_DEV_HOST} ${audius_client}`) + hostMappings.push(`${REMOTE_DEV_HOST} audius_client`) lines = [...lines, START_SENTINEL, ...hostMappings, END_SENTINEL, '\n'] writeArrayIntoFile(lines) }