diff --git a/packages/edge-gateway/src/durable-objects/summary-metrics.js b/packages/edge-gateway/src/durable-objects/summary-metrics.js index 7ae0b67..d024103 100644 --- a/packages/edge-gateway/src/durable-objects/summary-metrics.js +++ b/packages/edge-gateway/src/durable-objects/summary-metrics.js @@ -13,6 +13,7 @@ import { * @property {BigInt} totalCachedContentLengthBytes total content length of cached responses * @property {Record} contentLengthHistogram * @property {Record} responseTimeHistogram + * @property {Record} raceResponseTimeHistogram * * @typedef {Object} FetchStats * @property {number} responseTime number of milliseconds to get response @@ -35,11 +36,13 @@ const TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID = 'totalCachedContentLengthBytes' const CONTENT_LENGTH_HISTOGRAM_ID = 'contentLengthHistogram' // Key to track response time histogram const RESPONSE_TIME_HISTOGRAM_ID = 'responseTimeHistogram' +// Key to track race response time histogram +const RACE_RESPONSE_TIME_HISTOGRAM_ID = 'raceResponseTimeHistogram' /** * Durable Object for keeping summary metrics of nft.storage Gateway */ -export class SummaryMetrics0 { +export class SummaryMetrics1 { constructor(state) { this.state = state @@ -73,6 +76,10 @@ export class SummaryMetrics0 { this.responseTimeHistogram = (await this.state.storage.get(RESPONSE_TIME_HISTOGRAM_ID)) || createResponseTimeHistogramObject() + // Race response time histogram + this.raceResponseTimeHistogram = + (await this.state.storage.get(RACE_RESPONSE_TIME_HISTOGRAM_ID)) || + createResponseTimeHistogramObject() }) } @@ -96,6 +103,7 @@ export class SummaryMetrics0 { this.totalCachedContentLengthBytes.toString(), contentLengthHistogram: this.contentLengthHistogram, responseTimeHistogram: this.responseTimeHistogram, + raceResponseTimeHistogram: this.raceResponseTimeHistogram, }) ) default: @@ -127,7 +135,10 @@ export class SummaryMetrics0 { this.totalCachedResponses += 1 this.totalCachedContentLengthBytes += BigInt(stats.contentLength) this._updateContentLengthMetrics(stats) - this._updateResponseTimeHistogram(stats) + this.responseTimeHistogram = this._getUpdatedHistogram( + stats, + this.responseTimeHistogram + ) // Save updated metrics await Promise.all([ this.state.storage.put( @@ -165,7 +176,14 @@ export class SummaryMetrics0 { this.totalWinnerResponseTime += stats.responseTime this.totalWinnerSuccessfulRequests += 1 this._updateContentLengthMetrics(stats) - this._updateResponseTimeHistogram(stats) + this.responseTimeHistogram = this._getUpdatedHistogram( + stats, + this.responseTimeHistogram + ) + this.raceResponseTimeHistogram = this._getUpdatedHistogram( + stats, + this.raceResponseTimeHistogram + ) // Save updated Metrics await Promise.all([ this.state.storage.put( @@ -188,6 +206,10 @@ export class SummaryMetrics0 { RESPONSE_TIME_HISTOGRAM_ID, this.responseTimeHistogram ), + this.state.storage.put( + RACE_RESPONSE_TIME_HISTOGRAM_ID, + this.raceResponseTimeHistogram + ), ]) } @@ -215,10 +237,11 @@ export class SummaryMetrics0 { /** * @param {FetchStats} stats + * @param {Record} histogram */ - _updateResponseTimeHistogram(stats) { + _getUpdatedHistogram(stats, histogram) { const tmpHistogram = { - ...this.responseTimeHistogram, + ...histogram, } // Get all the histogram buckets where the response time is smaller @@ -230,7 +253,7 @@ export class SummaryMetrics0 { tmpHistogram[candidate] += 1 }) - this.responseTimeHistogram = tmpHistogram + return tmpHistogram } } diff --git a/packages/edge-gateway/src/index.js b/packages/edge-gateway/src/index.js index 3608f87..cf222b4 100644 --- a/packages/edge-gateway/src/index.js +++ b/packages/edge-gateway/src/index.js @@ -9,7 +9,7 @@ import { metricsGet } from './metrics.js' // Export Durable Object namespace from the root module. export { GatewayMetrics0 } from './durable-objects/gateway-metrics.js' -export { SummaryMetrics0 } from './durable-objects/summary-metrics.js' +export { SummaryMetrics1 } from './durable-objects/summary-metrics.js' export { CidsTracker0 } from './durable-objects/cids.js' export { GatewayRedirectCounter0 } from './durable-objects/gateway-redirect-counter.js' diff --git a/packages/edge-gateway/src/metrics.js b/packages/edge-gateway/src/metrics.js index 5d4d180..4f12d25 100644 --- a/packages/edge-gateway/src/metrics.js +++ b/packages/edge-gateway/src/metrics.js @@ -133,6 +133,15 @@ export async function metricsGet(request, env, ctx) { metricsCollected.summaryMetrics.totalWinnerSuccessfulRequests + metricsCollected.summaryMetrics.totalCachedResponses }`, + `# HELP nftgateway_race_responses_per_time_total total of responses per gateway race response time bucket`, + `# TYPE nftgateway_race_responses_per_time_total histogram`, + ...responseTimeHistogram.map( + (t) => + `nftgateway_race_responses_per_time_total{le="${msToS(t)}",env="${ + env.ENV + }"} ${metricsCollected.summaryMetrics.raceResponseTimeHistogram[t]}` + ), + `nftgateway_race_responses_per_time_total{le="+Inf",env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalWinnerSuccessfulRequests}`, `# HELP nftgateway_response_time_seconds_total Accumulated response time of each gateway.`, `# TYPE nftgateway_response_time_seconds_total summary`, ...env.ipfsGateways.map( diff --git a/packages/edge-gateway/wrangler.toml b/packages/edge-gateway/wrangler.toml index 8ce00a3..ca64895 100644 --- a/packages/edge-gateway/wrangler.toml +++ b/packages/edge-gateway/wrangler.toml @@ -19,7 +19,7 @@ main = "worker.mjs" [durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"}, {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] @@ -42,7 +42,7 @@ ENV = "production" [env.production.durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"}, {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] @@ -65,7 +65,7 @@ ENV = "staging" [env.staging.durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"}, {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] @@ -84,7 +84,7 @@ ENV = "test" [env.test.durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"}, {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] @@ -102,7 +102,7 @@ IPFS_GATEWAYS = "[\"https://ipfs.io\"]" [env.vsantos.durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"}, {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] @@ -119,7 +119,7 @@ IPFS_GATEWAYS = "[\"https://ipfs.io\"]" [env.alanshaw.durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"}, {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] @@ -145,7 +145,10 @@ deleted_classes = ["GatewayRateLimits2"] tag = "v5" # Should be unique for each entry new_classes = ["GatewayRateLimits4"] deleted_classes = ["GatewayRateLimits3"] - [[migrations]] tag = "v6" # Should be unique for each entry -deleted_classes = ["GatewayRateLimits4"] \ No newline at end of file +deleted_classes = ["GatewayRateLimits4"] +[[migrations]] +tag = "v7" # Should be unique for each entry +new_classes = ["SummaryMetrics1"] +deleted_classes = ["SummaryMetrics0"] \ No newline at end of file