Check all gateways if content cant be found in user replica set and I…

…PFS (#2024)

* Check all gateways if content cant be found in user replica set and IPFS

* Exclude gateways that have been tried

* Should be not includes

* Handle not found state correctly

* Remove log

creator-node/src/fileManager.js

@@ -10,6 +10,7 @@ const DiskManager = require('./diskManager')
 const { logger: genericLogger } = require('./logging')
 const { sendResponse, errorResponseBadRequest } = require('./apiHelpers')
 const ipfsAdd = require('./ipfsAdd')
+const { findCIDInNetwork } = require('./utils')
 
 const MAX_AUDIO_FILE_SIZE = parseInt(config.get('maxAudioFileSizeBytes')) // Default = 250,000,000 bytes = 250MB
 const MAX_MEMORY_FILE_SIZE = parseInt(config.get('maxMemoryFileSizeBytes')) // Default = 50,000,000 bytes = 50MB
@@ -270,6 +271,22 @@ async function saveFileForMultihashToFS (serviceRegistry, logger, multihash, exp
       }
     }
 
+    // if not found in gateways or IPFS, check nodes on the rest of the network
+    if (!fileFound) {
+      try {
+        const libs = serviceRegistry.libs
+        const found = await findCIDInNetwork(expectedStoragePath, multihash, logger, libs, null, gatewaysToTry)
+        if (found) {
+          decisionTree.push({ stage: `Found file ${multihash} by calling "findCIDInNetwork"`, vals: multihash, time: Date.now() })
+          fileFound = true
+        } else {
+          decisionTree.push({ stage: `Failed to retrieve file for multihash ${multihash} by calling findCIDInNetwork`, vals: multihash, time: Date.now() })
+        }
+      } catch (e) {
+        decisionTree.push({ stage: `Failed to retrieve file for multihash ${multihash} by calling findCIDInNetwork`, vals: multihash, time: Date.now() })
+      }
+    }
+
     // error if file was not found on any gateway or ipfs
     if (!fileFound) {
       const retrievalSourcesString = (SaveFileForMultihashToFSIPFSFallback) ? 'ipfs & other creator node gateways' : 'creator node gateways'
@@ -292,7 +309,6 @@ async function saveFileForMultihashToFS (serviceRegistry, logger, multihash, exp
       decisionTree.push({ stage: `Error during content verification for multihash`, vals: multihash, time: Date.now() })
       throw new Error(`Error during content verification for multihash ${multihash} ${e.message}`)
     }
-
     // If error, return boolean failure indicator + print logs
   } catch (e) {
     decisionTree.push({ stage: `saveFileForMultihashToFS error`, vals: e.message, time: Date.now() })

creator-node/src/utils.js

@@ -227,21 +227,36 @@ const ipfsGet = ({ ipfsLatest }, logger, path, timeout = 1000) => new Promise(as
   }
 })
 
-async function findCIDInNetwork (filePath, cid, logger, libs, trackId = null) {
+/**
+ *
+ * @param {String} filePath location of the file on disk
+ * @param {String} cid content hash of the file
+ * @param {Object} logger logger object
+ * @param {Object} libs libs instance
+ * @param {Integer?} trackId optional trackId that corresponds to the cid, see file_lookup route for more info
+ * @param {Array?} excludeList optional array of content nodes to exclude in network wide search
+ * @returns {Boolean} returns true if the file was found in the network
+ */
+async function findCIDInNetwork (filePath, cid, logger, libs, trackId = null, excludeList = []) {
+  let found = false
+
   const attemptedStateFix = await getIfAttemptedStateFix(filePath)
   if (attemptedStateFix) return
 
   // get list of creator nodes
   const creatorNodes = await getAllRegisteredCNodes(libs)
   if (!creatorNodes.length) return
 
+  // Remove excluded nodes from list of creator nodes, no-op if empty list or nothing passed in
+  const creatorNodesFiltered = creatorNodes.filter(c => !excludeList.includes(c.endpoint))
+
   // generate signature
   const delegateWallet = config.get('delegateOwnerWallet').toLowerCase()
   const { signature, timestamp } = generateTimestampAndSignature({ filePath, delegateWallet }, config.get('delegatePrivateKey'))
   let node
 
-  for (let index = 0; index < creatorNodes.length; index++) {
-    node = creatorNodes[index]
+  for (let index = 0; index < creatorNodesFiltered.length; index++) {
+    node = creatorNodesFiltered[index]
     try {
       const resp = await axios({
         method: 'get',
@@ -266,7 +281,7 @@ async function findCIDInNetwork (filePath, cid, logger, libs, trackId = null) {
           await fs.unlink(filePath)
           logger.error(`findCIDInNetwork - File contents don't match IPFS hash cid: ${cid} result: ${ipfsHashOnly}`)
         }
-
+        found = true
         logger.info(`findCIDInNetwork - successfully fetched file ${filePath} from node ${node.endpoint}`)
         break
       }
@@ -277,6 +292,8 @@ async function findCIDInNetwork (filePath, cid, logger, libs, trackId = null) {
       continue
     }
   }
+
+  return found
 }
 
 /**