Add pagination to script (#2080)

* Add pagination to script

* Refactor into a const

* Add endpoint to verification log

creator-node/scripts/updateContentBlacklist.js

@@ -22,6 +22,7 @@ const TYPES_SET = new Set(TYPES_ARR)
 
 const REQUEST_CONCURRENCY_LIMIT = 20
 const MAX_LIMIT = 500
+const VALUES_BATCH_SIZE = 10
 
 // Script usage:
 // node updateContentBlacklist.js -a add -l 1,3,7 -t user
@@ -50,40 +51,43 @@ async function run () {
     return
   }
 
-  console.log('Updating Content Blacklist...\n')
+  console.log(`Updating Content Blacklist for ${CREATOR_NODE_ENDPOINT}...\n`)
   const { action, type, values, verbose } = args
-  try {
-    switch (action) {
-      case 'ADD': {
-        await addToContentBlacklist(type, values)
-        break
-      }
-      case 'REMOVE': {
-        await removeFromContentBlacklist(type, values)
-        break
-      }
-      default: {
-        console.error('Should not have reached here :(')
-        return
+  for (let i = 0; i < values.length; i += VALUES_BATCH_SIZE) {
+    const valuesSliced = values.slice(i, i + VALUES_BATCH_SIZE)
+    try {
+      switch (action) {
+        case 'ADD': {
+          await addToContentBlacklist(type, valuesSliced)
+          break
+        }
+        case 'REMOVE': {
+          await removeFromContentBlacklist(type, valuesSliced)
+          break
+        }
+        default: {
+          console.error('Should not have reached here :(')
+          return
+        }
       }
+    } catch (e) {
+      console.error(`Failed to perform [${action}] for [${type}]: ${e}`)
+      return
     }
-  } catch (e) {
-    console.error(`Failed to perform [${action}] for [${type}]: ${e}`)
-    return
-  }
 
-  console.log('Verifying content against blacklist...\n')
-  try {
-    const segments = await verifyWithBlacklist({ type, values, action })
+    console.log(`Verifying content against blacklist for ${CREATOR_NODE_ENDPOINT}...\n`)
+    try {
+      const segments = await verifyWithBlacklist({ type, values: valuesSliced, action })
 
-    let successMsg = `Successfully performed [${action}] for type [${type}]!\nValues: [${values}]`
-    if (verbose) {
-      successMsg += `\nNumber of Segments: ${segments.length}`
-      successMsg += `\nSegments: ${segments}`
+      let successMsg = `Successfully performed [${action}] for type [${type}]!\values: [${valuesSliced}]`
+      if (verbose) {
+        successMsg += `\nNumber of Segments: ${segments.length}`
+        successMsg += `\nSegments: ${segments}`
+      }
+      console.log(successMsg)
+    } catch (e) {
+      console.error(`Verification check failed: ${e}`)
     }
-    console.log(successMsg)
-  } catch (e) {
-    console.error(`Verification check failed: ${e}`)
   }
 }
 
@@ -144,12 +148,14 @@ async function addToContentBlacklist (type, values) {
 
   let resp
   try {
-    resp = await axios({
+    const reqObj = {
       url: `${CREATOR_NODE_ENDPOINT}/blacklist/add`,
       method: 'post',
       params: { type, values, timestamp, signature },
       responseType: 'json'
-    })
+    }
+    console.log(`About to send axios request to ${reqObj.url} for values`, values)
+    resp = await axios(reqObj)
   } catch (e) {
     throw new Error(`Error with adding type [${type}] and values [${values}] to ContentBlacklist: ${e}`)
   }