Add Batch CID route to creator node (#1324)

* Add script to get all user clock values * Batch requests to get CID propogation * Make changes to creator node to batch ipfs checks * Make the script faster (?) * Remove semicolons and format code * Check for file existence * Check for cover and profile photos * Check for existence * Handle errors better and reduce timeouts * Have more readable json output * Address PR comments: * Get total number of users and tracks * Change default discovery URL * Rename variables for clarity * Add Req Limiter * Address most PR comments * Fix config val error * Fix CID script * Move await out * Delete script * Address PR comments: * check if cids exists * Use let instead of const * Use raw: true to make it more performant * Keep support for dir going forward * Change limit to 1000 * Name the variable better * Fix docstring error * Update limits on requests
AudiusProject · Mar 19, 2021 · c120330 · c120330
1 parent ad50b98
commit c120330
Show file tree

Hide file tree

Showing 7 changed files with 71 additions and 1 deletion.
diff --git a/creator-node/compose/env/base.env b/creator-node/compose/env/base.env
@@ -14,6 +14,7 @@ rateLimitingUserReqLimit=3000
 rateLimitingMetadataReqLimit=3000
 rateLimitingImageReqLimit=6000
 rateLimitingTrackReqLimit=6000
+rateLimitingBatchCidsExistLimit=1
 maxAudioFileSizeBytes=250000000
 maxMemoryFileSizeBytes=50000000
 serviceLatitude=

diff --git a/creator-node/default-config.json b/creator-node/default-config.json
@@ -15,6 +15,7 @@
   "rateLimitingMetadataReqLimit": 3000,
   "rateLimitingImageReqLimit": 6000,
   "rateLimitingTrackReqLimit": 6000,
+  "rateLimitingBatchCidsExistLimit": 1,
   "URSMRequestForSignatureReqLimit": 30,
   "endpointRateLimits": "{\"/image_upload\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/login\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/login/challenge\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/logout\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/batch_clock_status\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/track_content\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/tracks/metadata\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/tracks\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/audius_users/metadata\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/audius_users\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/sync\":{\"post\":[{\"expiry\":60,\"max\":500}]},\"/vector_clock_sync\":{\"post\":[{\"expiry\":60,\"max\":500}]}}",
   "maxAudioFileSizeBytes":1000000000,

diff --git a/creator-node/docker-compose/development.env b/creator-node/docker-compose/development.env
@@ -18,6 +18,7 @@ rateLimitingUserReqLimit=3000
 rateLimitingMetadataReqLimit=3000
 rateLimitingImageReqLimit=6000
 rateLimitingTrackReqLimit=6000
+rateLimitingBatchCidsExistLimit=1
 maxAudioFileSizeBytes=1000000000
 maxMemoryFileSizeBytes=50000000
 serviceLatitude=

diff --git a/creator-node/src/app.js b/creator-node/src/app.js
@@ -14,6 +14,7 @@ const {
   metadataReqLimiter,
   imageReqLimiter,
   URSMRequestForSignatureReqLimiter,
+  batchCidsExistReqLimiter,
   getRateLimiterMiddleware
 } = require('./reqLimiter')
 const config = require('./config')
@@ -37,6 +38,7 @@ app.use('/audius_user/', audiusUserReqLimiter)
 app.use('/metadata', metadataReqLimiter)
 app.use('/image_upload', imageReqLimiter)
 app.use('/ursm_request_for_signature', URSMRequestForSignatureReqLimiter)
+app.use('/batch_cids_exist', batchCidsExistReqLimiter)
 app.use(getRateLimiterMiddleware())
 
 // import routes

diff --git a/creator-node/src/config.js b/creator-node/src/config.js
@@ -183,6 +183,12 @@ const config = convict({
     env: 'rateLimitingTrackReqLimit',
     default: null
   },
+  rateLimitingBatchCidsExistLimit: {
+    doc: 'Total requests per hour rate limit for /track routes',
+    format: 'nat',
+    env: 'rateLimitingBatchCidsExistLimit',
+    default: null
+  },
   URSMRequestForSignatureReqLimit: {
     doc: 'Total requests per hour rate limit for /ursm_request_for_signature route',
     format: 'nat',

diff --git a/creator-node/src/reqLimiter.js b/creator-node/src/reqLimiter.js
@@ -95,6 +95,16 @@ const URSMRequestForSignatureReqLimiter = rateLimit({
   keyGenerator: ipKeyGenerator
 })
 
+const batchCidsExistReqLimiter = rateLimit({
+  store: new RedisStore({
+    client: client,
+    prefix: 'batchCidsExistLimit',
+    expiry: 5 // 5 seconds
+  }),
+  max: config.get('rateLimitingBatchCidsExistLimit'), // max requests every five seconds
+  keyGenerator: ipKeyGenerator
+})
+
 const onLimitReached = (req, res, options) => {
   req.logger.warn(req.rateLimit, `Rate Limit Hit`)
 }
@@ -162,5 +172,6 @@ module.exports = {
   metadataReqLimiter,
   imageReqLimiter,
   URSMRequestForSignatureReqLimiter,
+  batchCidsExistReqLimiter,
   getRateLimiterMiddleware
 }
diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js
@@ -29,7 +29,7 @@ const {
   triggerSecondarySyncs,
   ensureStorageMiddleware
 } = require('../middlewares')
-const { getIPFSPeerId, ipfsSingleByteCat, ipfsStat, getAllRegisteredCNodes, findCIDInNetwork } = require('../utils')
+const { getIPFSPeerId, ipfsSingleByteCat, ipfsStat, getAllRegisteredCNodes, findCIDInNetwork, timeout } = require('../utils')
 const ImageProcessingQueue = require('../ImageProcessingQueue')
 const RehydrateIpfsQueue = require('../RehydrateIpfsQueue')
 const DBManager = require('../dbManager')
@@ -39,6 +39,8 @@ const { promisify } = require('util')
 const fsStat = promisify(fs.stat)
 
 const FILE_CACHE_EXPIRY_SECONDS = 5 * 60
+const BATCH_CID_ROUTE_LIMIT = 500
+const BATCH_CID_EXISTS_CONCURRENCY_LIMIT = 50
 
 /**
  * Helper method to stream file from file system on creator node
@@ -490,6 +492,52 @@ module.exports = function (app) {
    */
   app.get('/ipfs/:dirCID/:filename', getDirCID)
 
+  /**
+   * Serves information on existence of given cids
+   * @param req
+   * @param req.body
+   * @param {string[]} req.body.cids the cids to check existence for, these cids can also be directories
+   * @dev This route can have a large number of CIDs as input, therefore we use a POST request.
+   */
+  app.post('/batch_cids_exist', handleResponse(async (req, res) => {
+    const { cids } = req.body
+
+    if (cids && cids.length > BATCH_CID_ROUTE_LIMIT) {
+      return errorResponseBadRequest(`Too many CIDs passed in, limit is ${BATCH_CID_ROUTE_LIMIT}`)
+    }
+
+    const queryResults = (await models.File.findAll({
+      attributes: ['multihash', 'storagePath'],
+      raw: true,
+      where: {
+        multihash: {
+          [models.Sequelize.Op.in]: cids
+        }
+      }
+    }))
+
+    let cidExists = {}
+
+    // Check if hash exists in disk in batches (to limit concurrent load)
+    for (let i = 0; i < queryResults.length; i += BATCH_CID_EXISTS_CONCURRENCY_LIMIT) {
+      const batch = queryResults.slice(i, i + BATCH_CID_EXISTS_CONCURRENCY_LIMIT)
+      const exists = await Promise.all(batch.map(
+        (storagePath) => fs.pathExists(storagePath)
+      ))
+      batch.map(({ multihash }, idx) => {
+        cidExists[multihash] = exists[idx]
+      })
+
+      await timeout(250)
+    }
+
+    const cidExistanceMap = {
+      cids: cids.map(cid => ({ cid, exists: cidExists[cid] || false }))
+    }
+
+    return successResponse(cidExistanceMap)
+  }))
+
   /**
    * Serve file from FS given a storage path
    * This is a cnode-cnode only route, not to be consumed by clients. It has auth restrictions to only