Consume SecondarySyncRequestOutcomes in Snapback reconfig (#1614)

AudiusProject · Jul 9, 2021 · d3e2a05 · d3e2a05
1 parent 9752351
commit d3e2a05
Show file tree

Hide file tree

Showing 5 changed files with 249 additions and 183 deletions.
diff --git a/creator-node/compose/env/base.env b/creator-node/compose/env/base.env
@@ -1,3 +1,4 @@
+# Node
 storagePath=/file_storage
 awsBucket=
 awsAccessKeyId=
@@ -9,6 +10,18 @@ port=4000
 logLevel=debug
 redisHost=
 redisPort=6379
+
+# Run as creator node by default
+isUserMetadataNode=false
+
+# Put CNs to debug mode if we're running locally.
+# Can be overriden.
+creatorNodeIsDebug=true
+
+WAIT_HOSTS=
+enableRehydrate=true
+
+# Rate limiting
 rateLimitingAudiusUserReqLimit=3000
 rateLimitingUserReqLimit=3000
 rateLimitingMetadataReqLimit=3000
@@ -27,29 +40,17 @@ delegatePrivateKey=
 dataProviderUrl=http://audius_ganache_cli:8545
 spOwnerWalletIndex=
 
-# Run as creator node by default
-isUserMetadataNode=false
-
-# Put CNs to debug mode if we're running locally.
-# Can be overriden.
-creatorNodeIsDebug=true
 
-WAIT_HOSTS=
-enableRehydrate=true
-
-# peerSetManager configs
-peerHealthCheckRequestTimeout=2000
-# bytes; 10gb
-minimumStoragePathSize=10000000000
-# bytes; 2gb 
-minimumMemoryAvailable=2000000000
-maxFileDescriptorsAllocatedPercentage=95
+# Sync / SnapbackSM configs
+snapbackReconfigEnabled=true
+snapbackJobInterval=10000 # ms
+snapbackModuloBase=2
 minimumDailySyncCount=5
 minimumRollingSyncCount=10
 minimumSuccessfulSyncCountPercentage=50
-
-# snapbackSM configs
-snapbackReconfigEnabled=true
-# ms
-snapbackJobInterval=3000
-snapbackModuloBase=2
+minimumSecondaryUserSyncSuccessPercent=50
+# peerSetManager
+peerHealthCheckRequestTimeout=2000 # ms
+minimumStoragePathSize=10000000000 # bytes; 10gb
+minimumMemoryAvailable=2000000000 # bytes; 2gb
+maxFileDescriptorsAllocatedPercentage=95
diff --git a/creator-node/default-config.json b/creator-node/default-config.json
@@ -10,6 +10,7 @@
   "logLevel": "debug",
   "redisHost": "localhost",
   "redisPort": 4379,
+
   "rateLimitingAudiusUserReqLimit": 3000,
   "rateLimitingUserReqLimit": 3000,
   "rateLimitingMetadataReqLimit": 3000,
@@ -18,6 +19,7 @@
   "rateLimitingBatchCidsExistLimit": 1,
   "URSMRequestForSignatureReqLimit": 30,
   "endpointRateLimits": "{\"/image_upload\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/login\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/login/challenge\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/logout\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/users/batch_clock_status\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/track_content\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/tracks/metadata\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/tracks\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/audius_users/metadata\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/audius_users\":{\"post\":[{\"expiry\":60,\"max\":100}]},\"/sync\":{\"post\":[{\"expiry\":60,\"max\":500}]},\"/vector_clock_sync\":{\"post\":[{\"expiry\":60,\"max\":500}]}}",
+
   "maxAudioFileSizeBytes":1000000000,
   "maxMemoryFileSizeBytes":50000000,
   "serviceLatitude": "",
@@ -42,7 +44,8 @@
   "dataNetworkId": "",
   "userMetadataNodeUrl": "http://docker.for.mac.localhost:4000",
   "rehydrateMaxConcurrency": 10,
+  "creatorNodeEndpoint": "",
+
   "nodeSyncFileSaveMaxConcurrency": 10,
-  "syncQueueMaxConcurrency": 50,
-  "creatorNodeEndpoint": ""
+  "syncQueueMaxConcurrency": 50
 }
diff --git a/creator-node/src/config.js b/creator-node/src/config.js
@@ -126,14 +126,9 @@ const config = convict({
     default: null
   },
 
-  maxExportClockValueRange: {
-    doc: 'Maximum range of clock values to export at once to prevent process OOM',
-    format: Number,
-    env: 'maxExportClockValueRange',
-    default: 10000
-  },
-
-  // Rate limit configs
+  /**
+   * Rate limit configs
+   */
   endpointRateLimits: {
     doc: `A serialized objects of rate limits with the form {
       <req.path>: {
@@ -273,32 +268,6 @@ const config = convict({
     default: -1
   },
 
-  // Sync configs
-  nodeSyncFileSaveMaxConcurrency: {
-    doc: 'Max concurrency of saveFileForMultihashToFS calls inside nodesync',
-    format: 'nat',
-    env: 'nodeSyncFileSaveMaxConcurrency',
-    default: 10
-  },
-  syncQueueMaxConcurrency: {
-    doc: 'Max concurrency of SyncQueue',
-    format: 'nat',
-    env: 'syncQueueMaxConcurrency',
-    default: 50
-  },
-  issueAndWaitForSecondarySyncRequestsPollingDurationMs: {
-    doc: 'Duration for which to poll secondaries for content replication in `issueAndWaitForSecondarySyncRequests` function',
-    format: 'nat',
-    env: 'issueAndWaitForSecondarySyncRequestsPollingDurationMs',
-    default: 5000 // 5000ms = 5s (prod default)
-  },
-  enforceWriteQuorum: {
-    doc: 'Boolean flag indicating whether or not primary should reject write on 2/3 replication across replica set',
-    format: Boolean,
-    env: 'enforceWriteQuorum',
-    default: false
-  },
-
   // wallet information
   delegateOwnerWallet: {
     doc: 'wallet address',
@@ -391,12 +360,6 @@ const config = convict({
     env: 'userMetadataNodeUrl',
     default: ''
   },
-  debounceTime: {
-    doc: 'sync debounce time in ms',
-    format: 'nat',
-    env: 'debounceTime',
-    default: 0 // 0ms
-  },
   dataRegistryAddress: {
     doc: 'data contracts registry address',
     format: String,
@@ -421,8 +384,6 @@ const config = convict({
     env: 'creatorNodeEndpoint',
     default: null
   },
-
-  // Service selection
   discoveryProviderWhitelist: {
     doc: 'Whitelisted discovery providers to select from (comma-separated)',
     format: String,
@@ -447,30 +408,6 @@ const config = convict({
     env: 'rehydrateMaxConcurrency',
     default: 10
   },
-  snapbackDevModeEnabled: {
-    doc: 'TEST ONLY. DO NOT CONFIGURE MANUALLY. Disables automatic secondary sync issuing in order to test SnapbackSM.',
-    format: 'BooleanCustom',
-    env: 'snapbackDevModeEnabled',
-    default: false
-  },
-  snapbackReconfigEnabled: {
-    doc: 'Enables replica set reconfiguration or not. Currently enabled for dev mode, disabled for prod.',
-    format: 'BooleanCustom',
-    env: 'snapbackReconfigEnabled',
-    default: false
-  },
-  snapbackModuloBase: {
-    doc: 'The modulo base to segment users by on snapback. Will process `1/snapbackModuloBase` users at some snapback interval',
-    format: 'nat',
-    env: 'snapbackModuloBase',
-    default: 24
-  },
-  snapbackJobInterval: {
-    doc: 'Interval [ms] that snapbackSM jobs are fired; 1 hour',
-    format: 'nat',
-    env: 'snapbackJobInterval',
-    default: 3600000
-  },
   devMode: {
     doc: 'Used to differentiate production vs dev mode for node',
     format: 'BooleanCustom',
@@ -501,6 +438,69 @@ const config = convict({
     env: 'enableRehydrate',
     default: true
   },
+
+  /** sync / snapback configs */
+
+  debounceTime: {
+    doc: 'sync debounce time in ms',
+    format: 'nat',
+    env: 'debounceTime',
+    default: 0 // 0ms
+  },
+  maxExportClockValueRange: {
+    doc: 'Maximum range of clock values to export at once to prevent process OOM',
+    format: Number,
+    env: 'maxExportClockValueRange',
+    default: 10000
+  },
+  nodeSyncFileSaveMaxConcurrency: {
+    doc: 'Max concurrency of saveFileForMultihashToFS calls inside nodesync',
+    format: 'nat',
+    env: 'nodeSyncFileSaveMaxConcurrency',
+    default: 10
+  },
+  syncQueueMaxConcurrency: {
+    doc: 'Max concurrency of SyncQueue',
+    format: 'nat',
+    env: 'syncQueueMaxConcurrency',
+    default: 50
+  },
+  issueAndWaitForSecondarySyncRequestsPollingDurationMs: {
+    doc: 'Duration for which to poll secondaries for content replication in `issueAndWaitForSecondarySyncRequests` function',
+    format: 'nat',
+    env: 'issueAndWaitForSecondarySyncRequestsPollingDurationMs',
+    default: 5000 // 5000ms = 5s (prod default)
+  },
+  enforceWriteQuorum: {
+    doc: 'Boolean flag indicating whether or not primary should reject write on 2/3 replication across replica set',
+    format: Boolean,
+    env: 'enforceWriteQuorum',
+    default: false
+  },
+  snapbackDevModeEnabled: {
+    doc: 'TEST ONLY. DO NOT CONFIGURE MANUALLY. Disables automatic secondary sync issuing in order to test SnapbackSM.',
+    format: 'BooleanCustom',
+    env: 'snapbackDevModeEnabled',
+    default: false
+  },
+  snapbackReconfigEnabled: {
+    doc: 'Enables replica set reconfiguration or not. Currently enabled for dev mode, disabled for prod.',
+    format: 'BooleanCustom',
+    env: 'snapbackReconfigEnabled',
+    default: false
+  },
+  snapbackModuloBase: {
+    doc: 'The modulo base to segment users by on snapback. Will process `1/snapbackModuloBase` users at some snapback interval',
+    format: 'nat',
+    env: 'snapbackModuloBase',
+    default: 24
+  },
+  snapbackJobInterval: {
+    doc: 'Interval [ms] that snapbackSM jobs are fired; 1 hour',
+    format: 'nat',
+    env: 'snapbackJobInterval',
+    default: 3600000
+  },
   maxManualRequestSyncJobConcurrency: {
     doc: 'Max bull queue concurrency for manual sync request jobs',
     format: 'nat',
@@ -513,8 +513,6 @@ const config = convict({
     env: 'maxRecurringRequestSyncJobConcurrency',
     default: 5
   },
-
-  // peerSetManager configs
   peerHealthCheckRequestTimeout: {
     doc: 'Timeout [ms] for checking health check route',
     format: 'nat',
@@ -552,14 +550,23 @@ const config = convict({
     default: 5000
   },
   minimumSuccessfulSyncCountPercentage: {
-    doc: 'Minimum percentage of failed syncs to be considered unhealthy',
+    doc: 'Minimum percentage of failed syncs to be considered healthy',
     format: 'nat',
     env: 'minimumSuccessfulSyncCountPercentage',
     // TODO: Update to higher percentage when higher threshold of syncs are passing
     default: 0
+  },
+  minimumSecondaryUserSyncSuccessPercent: {
+    doc: 'Minimum percent of failed Syncs for a user on a secondary for the secondary to be considered healthy for that user',
+    format: 'nat',
+    env: 'minimumSecondaryUserSyncSuccessPercent',
+    default: 50
   }
 
-  // unsupported options at the moment
+  /**
+   * unsupported options at the moment
+   */
+
   // awsBucket: {
   //   doc: 'AWS S3 bucket to upload files to',
   //   format: String,