feat: track histogram of race response times (#79)

nftstorage · May 12, 2022 · 7ee39e4 · 7ee39e4
1 parent f3e107c
commit 7ee39e4
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 15 deletions.
diff --git a/packages/edge-gateway/src/durable-objects/summary-metrics.js b/packages/edge-gateway/src/durable-objects/summary-metrics.js
@@ -13,6 +13,7 @@ import {
  * @property {BigInt} totalCachedContentLengthBytes total content length of cached responses
  * @property {Record<string, number>} contentLengthHistogram
  * @property {Record<string, number>} responseTimeHistogram
+ * @property {Record<string, number>} raceResponseTimeHistogram
  *
  * @typedef {Object} FetchStats
  * @property {number} responseTime number of milliseconds to get response
@@ -35,11 +36,13 @@ const TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID = 'totalCachedContentLengthBytes'
 const CONTENT_LENGTH_HISTOGRAM_ID = 'contentLengthHistogram'
 // Key to track response time histogram
 const RESPONSE_TIME_HISTOGRAM_ID = 'responseTimeHistogram'
+// Key to track race response time histogram
+const RACE_RESPONSE_TIME_HISTOGRAM_ID = 'raceResponseTimeHistogram'
 
 /**
  * Durable Object for keeping summary metrics of nft.storage Gateway
  */
-export class SummaryMetrics0 {
+export class SummaryMetrics1 {
   constructor(state) {
     this.state = state
 
@@ -73,6 +76,10 @@ export class SummaryMetrics0 {
       this.responseTimeHistogram =
         (await this.state.storage.get(RESPONSE_TIME_HISTOGRAM_ID)) ||
         createResponseTimeHistogramObject()
+      // Race response time histogram
+      this.raceResponseTimeHistogram =
+        (await this.state.storage.get(RACE_RESPONSE_TIME_HISTOGRAM_ID)) ||
+        createResponseTimeHistogramObject()
     })
   }
 
@@ -96,6 +103,7 @@ export class SummaryMetrics0 {
                 this.totalCachedContentLengthBytes.toString(),
               contentLengthHistogram: this.contentLengthHistogram,
               responseTimeHistogram: this.responseTimeHistogram,
+              raceResponseTimeHistogram: this.raceResponseTimeHistogram,
             })
           )
         default:
@@ -127,7 +135,10 @@ export class SummaryMetrics0 {
     this.totalCachedResponses += 1
     this.totalCachedContentLengthBytes += BigInt(stats.contentLength)
     this._updateContentLengthMetrics(stats)
-    this._updateResponseTimeHistogram(stats)
+    this.responseTimeHistogram = this._getUpdatedHistogram(
+      stats,
+      this.responseTimeHistogram
+    )
     // Save updated metrics
     await Promise.all([
       this.state.storage.put(
@@ -165,7 +176,14 @@ export class SummaryMetrics0 {
     this.totalWinnerResponseTime += stats.responseTime
     this.totalWinnerSuccessfulRequests += 1
     this._updateContentLengthMetrics(stats)
-    this._updateResponseTimeHistogram(stats)
+    this.responseTimeHistogram = this._getUpdatedHistogram(
+      stats,
+      this.responseTimeHistogram
+    )
+    this.raceResponseTimeHistogram = this._getUpdatedHistogram(
+      stats,
+      this.raceResponseTimeHistogram
+    )
     // Save updated Metrics
     await Promise.all([
       this.state.storage.put(
@@ -188,6 +206,10 @@ export class SummaryMetrics0 {
         RESPONSE_TIME_HISTOGRAM_ID,
         this.responseTimeHistogram
       ),
+      this.state.storage.put(
+        RACE_RESPONSE_TIME_HISTOGRAM_ID,
+        this.raceResponseTimeHistogram
+      ),
     ])
   }
 
@@ -215,10 +237,11 @@ export class SummaryMetrics0 {
 
   /**
    * @param {FetchStats} stats
+   * @param {Record<number, number>} histogram
    */
-  _updateResponseTimeHistogram(stats) {
+  _getUpdatedHistogram(stats, histogram) {
     const tmpHistogram = {
-      ...this.responseTimeHistogram,
+      ...histogram,
     }
 
     // Get all the histogram buckets where the response time is smaller
@@ -230,7 +253,7 @@ export class SummaryMetrics0 {
       tmpHistogram[candidate] += 1
     })
 
-    this.responseTimeHistogram = tmpHistogram
+    return tmpHistogram
   }
 }
 

diff --git a/packages/edge-gateway/src/index.js b/packages/edge-gateway/src/index.js
@@ -9,7 +9,7 @@ import { metricsGet } from './metrics.js'
 
 // Export Durable Object namespace from the root module.
 export { GatewayMetrics0 } from './durable-objects/gateway-metrics.js'
-export { SummaryMetrics0 } from './durable-objects/summary-metrics.js'
+export { SummaryMetrics1 } from './durable-objects/summary-metrics.js'
 export { CidsTracker0 } from './durable-objects/cids.js'
 export { GatewayRedirectCounter0 } from './durable-objects/gateway-redirect-counter.js'
 

diff --git a/packages/edge-gateway/src/metrics.js b/packages/edge-gateway/src/metrics.js
@@ -133,6 +133,15 @@ export async function metricsGet(request, env, ctx) {
       metricsCollected.summaryMetrics.totalWinnerSuccessfulRequests +
       metricsCollected.summaryMetrics.totalCachedResponses
     }`,
+    `# HELP nftgateway_race_responses_per_time_total total of responses per gateway race response time bucket`,
+    `# TYPE nftgateway_race_responses_per_time_total histogram`,
+    ...responseTimeHistogram.map(
+      (t) =>
+        `nftgateway_race_responses_per_time_total{le="${msToS(t)}",env="${
+          env.ENV
+        }"} ${metricsCollected.summaryMetrics.raceResponseTimeHistogram[t]}`
+    ),
+    `nftgateway_race_responses_per_time_total{le="+Inf",env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalWinnerSuccessfulRequests}`,
     `# HELP nftgateway_response_time_seconds_total Accumulated response time of each gateway.`,
     `# TYPE nftgateway_response_time_seconds_total summary`,
     ...env.ipfsGateways.map(

diff --git a/packages/edge-gateway/wrangler.toml b/packages/edge-gateway/wrangler.toml
@@ -19,7 +19,7 @@ main = "worker.mjs"
 [durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
@@ -42,7 +42,7 @@ ENV = "production"
 [env.production.durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
@@ -65,7 +65,7 @@ ENV = "staging"
 [env.staging.durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
@@ -84,7 +84,7 @@ ENV = "test"
 [env.test.durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
@@ -102,7 +102,7 @@ IPFS_GATEWAYS = "[\"https://ipfs.io\"]"
 [env.vsantos.durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
@@ -119,7 +119,7 @@ IPFS_GATEWAYS = "[\"https://ipfs.io\"]"
 [env.alanshaw.durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
@@ -145,7 +145,10 @@ deleted_classes = ["GatewayRateLimits2"]
 tag = "v5" # Should be unique for each entry
 new_classes = ["GatewayRateLimits4"]
 deleted_classes = ["GatewayRateLimits3"]
-
 [[migrations]]
 tag = "v6" # Should be unique for each entry
-deleted_classes = ["GatewayRateLimits4"]
+deleted_classes = ["GatewayRateLimits4"]
+[[migrations]]
+tag = "v7" # Should be unique for each entry
+new_classes = ["SummaryMetrics1"]
+deleted_classes = ["SummaryMetrics0"]