Content Node Chain of Trust [Phase 2] - automatic node self-registrat…

…ion on URSM (And new Content Node Init pattern) (#1232)
AudiusProject · Mar 11, 2021 · a440004 · a440004
1 parent 6590350
commit a440004
Show file tree

Hide file tree

Showing 22 changed files with 1,158 additions and 471 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -174,7 +174,7 @@ jobs:
             cd /home/circleci/project/service-commands/scripts/
             sudo "$(which node)" hosts.js add
             sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-            "$(which node)" setup.js up -nc 3
+            "$(which node)" setup.js up -nc 4
             cd /home/circleci/project/service-commands/scripts/
             "$(which node)" setup.js run user-replica-set-manager up
             cd /home/circleci/project/mad-dog/

diff --git a/creator-node/compose/docker-compose.yml b/creator-node/compose/docker-compose.yml
@@ -27,6 +27,7 @@ services:
       - delegatePrivateKey=${delegatePrivateKey}
       - creatorNodeEndpoint=${creatorNodeEndpoint}
       - snapbackDevModeEnabled=${snapbackDevModeEnabled}
+      - devMode=${DEV_MODE}
       - spOwnerWallet=${spOwnerWallet}
       - isUserMetadataNode=${CREATOR_NODE_IS_USER_METADATA}
     env_file:

diff --git a/creator-node/compose/env/commonEnv.sh b/creator-node/compose/env/commonEnv.sh
@@ -1 +1,2 @@
 export CREATOR_NODE_IS_USER_METADATA=false
+export DEV_MODE=true
diff --git a/creator-node/default-config.json b/creator-node/default-config.json
@@ -15,6 +15,7 @@
   "rateLimitingMetadataReqLimit": 3000,
   "rateLimitingImageReqLimit": 6000,
   "rateLimitingTrackReqLimit": 6000,
+  "URSMRequestForSignatureReqLimit": 30,
   "endpointRateLimits": "{\"/image_upload\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/login\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/login/challenge\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/logout\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/batch_clock_status\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/track_content\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/tracks/metadata\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/tracks\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/audius_users/metadata\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/audius_users\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/sync\":{\"post\":[{\"expiry\":60,\"max\":500}]},\"/vector_clock_sync\":{\"post\":[{\"expiry\":60,\"max\":500}]}}",
   "maxAudioFileSizeBytes":1000000000,
   "maxMemoryFileSizeBytes":50000000,

diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json
diff --git a/creator-node/package.json b/creator-node/package.json
@@ -18,7 +18,7 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "@audius/libs": "^1.1.9",
+    "@audius/libs": "1.1.16",
     "JSONStream": "^1.3.5",
     "axios": "^0.19.0",
     "base64-url": "^2.2.0",

diff --git a/creator-node/src/app.js b/creator-node/src/app.js
@@ -13,6 +13,7 @@ const {
   audiusUserReqLimiter,
   metadataReqLimiter,
   imageReqLimiter,
+  URSMRequestForSignatureReqLimiter,
   getRateLimiterMiddleware
 } = require('./reqLimiter')
 const config = require('./config')
@@ -35,6 +36,7 @@ app.use('/track*', trackReqLimiter)
 app.use('/audius_user/', audiusUserReqLimiter)
 app.use('/metadata', metadataReqLimiter)
 app.use('/image_upload', imageReqLimiter)
+app.use('/ursm_request_for_signature', URSMRequestForSignatureReqLimiter)
 app.use(getRateLimiterMiddleware())
 
 // import routes
@@ -50,11 +52,16 @@ function errorHandler (err, req, res, next) {
 }
 app.use(errorHandler)
 
+/**
+ * Configures express app object with required properties and starts express server
+ *
+ * @param {number} port port number on which to expose server
+ * @param {ServiceRegistry} serviceRegistry object housing all Content Node Services
+ */
 const initializeApp = (port, serviceRegistry) => {
   const storagePath = DiskManager.getConfigStoragePath()
 
-  // TODO: Can remove these when all routes
-  // consume serviceRegistry
+  // TODO: Can remove these when all routes consume serviceRegistry
   app.set('ipfsAPI', serviceRegistry.ipfs)
   app.set('storagePath', storagePath)
   app.set('redisClient', serviceRegistry.redis)

diff --git a/creator-node/src/config.js b/creator-node/src/config.js
@@ -183,6 +183,12 @@ const config = convict({
     env: 'rateLimitingTrackReqLimit',
     default: null
   },
+  URSMRequestForSignatureReqLimit: {
+    doc: 'Total requests per hour rate limit for /ursm_request_for_signature route',
+    format: 'nat',
+    env: 'URSMRequestForSignatureReqLimit',
+    default: null
+  },
 
   maxAudioFileSizeBytes: {
     doc: 'Maximum file size for audio file uploads in bytes',
@@ -409,6 +415,12 @@ const config = convict({
     env: 'snapbackDevModeEnabled',
     default: false
   },
+  devMode: {
+    doc: 'Used to differentiate production vs dev mode for node',
+    format: 'BooleanCustom',
+    env: 'devMode',
+    default: false
+  },
   maxStorageUsedPercent: {
     doc: 'Max percentage of storage capacity allowed to be used in CNode before blocking writes',
     format: 'nat',

diff --git a/creator-node/src/index.js b/creator-node/src/index.js
@@ -70,13 +70,19 @@ const startApp = async () => {
 
     await logIpfsPeerIds()
 
+    const nodeMode = config.get('devMode') ? 'Dev Mode' : 'Production Mode'
+
     await serviceRegistry.initServices()
-    logger.info('Initialized services')
+    logger.info(`Initialized services (Node running in ${nodeMode})`)
 
     appInfo = initializeApp(config.get('port'), serviceRegistry)
     logger.info('Initialized app and server')
 
     await pinCID(serviceRegistry.ipfsLatest)
+
+    // Some Services cannot start until server is up. Start them now
+    // No need to await on this as this process can take a while and can run in the background
+    serviceRegistry.initServicesThatRequireServer()
   }
 
   // when app terminates, close down any open DB connections gracefully

diff --git a/creator-node/src/reqLimiter.js b/creator-node/src/reqLimiter.js
@@ -85,6 +85,16 @@ const imageReqLimiter = rateLimit({
   keyGenerator: ipKeyGenerator
 })
 
+const URSMRequestForSignatureReqLimiter = rateLimit({
+  store: new RedisStore({
+    client: client,
+    prefix: 'URSMRequestForSignatureReqLimiter',
+    expiry: 60 * 60 // one hour in seconds
+  }),
+  max: config.get('URSMRequestForSignatureReqLimit'), // max requests per hour
+  keyGenerator: ipKeyGenerator
+})
+
 const onLimitReached = (req, res, options) => {
   req.logger.warn(req.rateLimit, `Rate Limit Hit`)
 }
@@ -151,5 +161,6 @@ module.exports = {
   audiusUserReqLimiter,
   metadataReqLimiter,
   imageReqLimiter,
+  URSMRequestForSignatureReqLimiter,
   getRateLimiterMiddleware
 }