Skip to content

Commit

Permalink
Add aggregator config to each metric (#3972)
Browse files Browse the repository at this point in the history
  • Loading branch information
@theoilie
theoilie committed Sep 29, 2022
1 parent 2106ccd commit 3d7018b
Showing 1 changed file with 40 additions and 18 deletions.
58 changes: 40 additions & 18 deletions creator-node/src/services/prometheusMonitoring/prometheus.constants.ts
View file
Expand Up @@ -209,12 +209,16 @@ const METRIC_LABEL_NAMES = Object.freeze(
)
)

type AggregatorType = 'sum' | 'first' | 'min' | 'max' | 'average' | 'omit'
type Metric = {
metricType: any
metricConfig: {
name: string
help: string
labelNames: string[]
// Function to aggregate metrics across workers.
// See https://github.com/siimon/prom-client/blob/96f7495d66b1a21755f745b1367d3e530668a957/lib/metricAggregators.js#L50
aggregator: AggregatorType
}
}

Expand All @@ -224,55 +228,62 @@ export const METRICS: Record<string, Metric> = Object.freeze({
metricConfig: {
name: METRIC_NAMES.RECOVER_ORPHANED_DATA_WALLET_COUNTS_GAUGE,
help: 'Number of wallets found with data orphaned on this node',
labelNames: []
labelNames: [],
aggregator: 'max' as AggregatorType
}
},
[METRIC_NAMES.RECOVER_ORPHANED_DATA_SYNC_COUNTS_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.RECOVER_ORPHANED_DATA_SYNC_COUNTS_GAUGE,
help: 'Number of syncs enqueued to recover data orphaned on this node',
labelNames: []
labelNames: [],
aggregator: 'max' as AggregatorType
}
},
[METRIC_NAMES.JOBS_COMPLETED_TOTAL_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.JOBS_COMPLETED_TOTAL_GAUGE,
help: 'Number of completed jobs',
labelNames: ['queue_name', 'job_name']
labelNames: ['queue_name', 'job_name'],
aggregator: 'first' as AggregatorType
}
},
[METRIC_NAMES.JOBS_FAILED_TOTAL_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.JOBS_FAILED_TOTAL_GAUGE,
help: 'Number of failed jobs',
labelNames: ['queue_name', 'job_name']
labelNames: ['queue_name', 'job_name'],
aggregator: 'first' as AggregatorType
}
},
[METRIC_NAMES.JOBS_DELAYED_TOTAL_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.JOBS_DELAYED_TOTAL_GAUGE,
help: 'Number of delayed jobs',
labelNames: ['queue_name', 'job_name']
labelNames: ['queue_name', 'job_name'],
aggregator: 'first' as AggregatorType
}
},
[METRIC_NAMES.JOBS_ACTIVE_TOTAL_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.JOBS_ACTIVE_TOTAL_GAUGE,
help: 'Number of active jobs',
labelNames: ['queue_name', 'job_name']
labelNames: ['queue_name', 'job_name'],
aggregator: 'first' as AggregatorType
}
},
[METRIC_NAMES.JOBS_WAITING_TOTAL_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.JOBS_WAITING_TOTAL_GAUGE,
help: 'Number of waiting jobs',
labelNames: ['queue_name', 'job_name']
labelNames: ['queue_name', 'job_name'],
aggregator: 'first' as AggregatorType
}
},
[METRIC_NAMES.JOBS_DURATION_SECONDS_HISTOGRAM]: {
Expand All @@ -282,17 +293,19 @@ export const METRICS: Record<string, Metric> = Object.freeze({
help: 'Time to complete jobs',
labelNames: ['queue_name', 'job_name', 'status'],
// 10 buckets in the range of 1 seconds to max to 10 minutes
buckets: exponentialBucketsRange(1, 600, 10)
buckets: exponentialBucketsRange(1, 600, 10),
aggregator: 'average' as AggregatorType
}
},
[METRIC_NAMES.JOBS_WAITING_DURATION_SECONDS_HISTROGRAM]: {
[METRIC_NAMES.JOBS_WAITING_DURATION_SECONDS_HISTOGRAM]: {
metricType: METRIC_TYPES.HISTOGRAM,
metricConfig: {
name: METRIC_NAMES.JOBS_WAITING_DURATION_SECONDS_HISTOGRAM,
help: 'Time spent waiting for jobs to run',
labelNames: ['queue_name', 'job_name', 'status'],
// 10 buckets in the range of 1 seconds to max to 10 minutes
buckets: exponentialBucketsRange(1, 600, 10)
buckets: exponentialBucketsRange(1, 600, 10),
aggregator: 'average' as AggregatorType
}
},
[METRIC_NAMES.JOBS_ATTEMPTS_HISTOGRAM]: {
Expand All @@ -302,7 +315,8 @@ export const METRICS: Record<string, Metric> = Object.freeze({
help: 'Job attempts made',
labelNames: ['queue_name', 'job_name', 'status'],
// 10 buckets in the range of 1 seconds to max to 10 minutes
buckets: exponentialBucketsRange(1, 600, 10)
buckets: exponentialBucketsRange(1, 600, 10),
aggregator: 'average' as AggregatorType
}
},
[METRIC_NAMES.SECONDARY_SYNC_FROM_PRIMARY_DURATION_SECONDS_HISTOGRAM]: {
Expand All @@ -313,15 +327,18 @@ export const METRICS: Record<string, Metric> = Object.freeze({
labelNames:
METRIC_LABEL_NAMES[
METRIC_NAMES.SECONDARY_SYNC_FROM_PRIMARY_DURATION_SECONDS_HISTOGRAM
]
],
aggregator: 'average' as AggregatorType
}
},
// TODO: This isn't used anywhere
[METRIC_NAMES.SYNC_QUEUE_JOBS_TOTAL_GAUGE]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.SYNC_QUEUE_JOBS_TOTAL_GAUGE,
help: 'Current job counts for SyncQueue by status',
labelNames: ['status']
labelNames: ['status'],
aggregator: 'first' as AggregatorType
}
},

Expand All @@ -339,7 +356,8 @@ export const METRICS: Record<string, Metric> = Object.freeze({
1,
config.get('maxSyncMonitoringDurationInMs') / 1000,
4
)
),
aggregator: 'max' as AggregatorType
}
},

Expand All @@ -364,7 +382,8 @@ export const METRICS: Record<string, Metric> = Object.freeze({
]
] || [])
],
buckets: [1, 5, 10, 30, 60, 120] // 1 second to 2 minutes
buckets: [1, 5, 10, 30, 60, 120], // 1 second to 2 minutes
aggregator: 'average' as AggregatorType
}
}
])
Expand All @@ -375,7 +394,8 @@ export const METRICS: Record<string, Metric> = Object.freeze({
name: METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE,
help: "Counts for each find-sync-requests job's result when looking for syncs that should be requested from a primary to a secondary",
labelNames:
METRIC_LABEL_NAMES[METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE]
METRIC_LABEL_NAMES[METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE],
aggregator: 'max' as AggregatorType
}
},
[METRIC_NAMES.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM]: {
Expand All @@ -393,15 +413,17 @@ export const METRICS: Record<string, Metric> = Object.freeze({
config.get('issueAndWaitForSecondarySyncRequestsPollingDurationMs') /
1000,
5
)
),
aggregator: 'average' as AggregatorType
}
},
[METRIC_NAMES.STORAGE_PATH_SIZE_BYTES]: {
metricType: METRIC_TYPES.GAUGE,
metricConfig: {
name: METRIC_NAMES.STORAGE_PATH_SIZE_BYTES,
help: 'Disk storage size',
labelNames: ['type']
labelNames: ['type'],
aggregator: 'first' as AggregatorType
}
}
})
Expand Down

0 comments on commit 3d7018b

Please sign in to comment.