Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

mixin: Add block deletion panels for compactor dashboard #3380

Merged
merged 2 commits into from Jan 5, 2021
Merged

mixin: Add block deletion panels for compactor dashboard #3380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aed289d
mixin: add deletion metrics for compactor
Sayrus Nov 13, 2020
044241f
Update CHANGELOG.md
kakkoyun Jan 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Expand Up @@ -209,6 +209,7 @@ sse_config:
- [#2579](https://github.com/thanos-io/thanos/pull/2579) Store: Experimental caching bucket can now cache metadata as well. Config has changed from #2532.
- [#2526](https://github.com/thanos-io/thanos/pull/2526) Compact: In case there are no labels left after deduplication via `--deduplication.replica-label`, assign first `replica-label` with value `deduped`.
- [#2621](https://github.com/thanos-io/thanos/pull/2621) Receive: Added flag to configure forward request timeout. Receive write will complete request as soon as quorum of writes succeeds.
- [#3380](https://github.com/thanos-io/thanos/pull/3380) mixin: Add block deletion panels for compactor dashboards.

### Changed

Expand Down
261 changes: 252 additions & 9 deletions examples/dashboards/compact.json
View file
Open in desktop
Expand Up @@ -616,7 +616,7 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"description": "Shows rate of execution for all meta files from blocks in the bucket into the memory.",
"description": "Shows deletion rate of blocks already marked for deletion.",
"fill": 10,
"id": 8,
"legend": {
Expand All @@ -641,6 +641,249 @@
"span": 4,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(thanos_compact_blocks_cleaned_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Blocks cleanup {{job}}",
"legendLink": null,
"step": 10
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Deletion Rate",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
},
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"description": "Shows deletion failures rate of blocks already marked for deletion.",
"fill": 1,
"id": 9,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [ ],
"spaceLength": 10,
"span": 4,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(thanos_compact_block_cleanup_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Blocks cleanup failures {{job}}",
"legendLink": null,
"step": 10
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Deletion Error Rate",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
},
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"description": "Shows rate at which blocks are marked for deletion (from GC and retention policy).",
"fill": 1,
"id": 10,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [ ],
"spaceLength": 10,
"span": 4,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(thanos_compact_blocks_marked_for_deletion_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Blocks marked {{job}}",
"legendLink": null,
"step": 10
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Marking Rate",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
"title": "Blocks deletion",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"description": "Shows rate of execution for all meta files from blocks in the bucket into the memory.",
"fill": 10,
"id": 11,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 0,
"links": [ ],
"nullPointMode": "null as zero",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [ ],
"spaceLength": 10,
"span": 4,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(thanos_blocks_meta_syncs_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
Expand Down Expand Up @@ -697,7 +940,7 @@
"datasource": "$datasource",
"description": "Shows ratio of errors compared to the total number of executed meta file sync.",
"fill": 10,
"id": 9,
"id": 12,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -774,7 +1017,7 @@
"datasource": "$datasource",
"description": "Shows how long has it taken to execute meta file sync, in quantiles.",
"fill": 1,
"id": 10,
"id": 13,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -879,7 +1122,7 @@
"datasource": "$datasource",
"description": "Shows rate of execution for operations against the bucket.",
"fill": 10,
"id": 11,
"id": 14,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -958,7 +1201,7 @@
"datasource": "$datasource",
"description": "Shows ratio of errors compared to the total number of executed operations against the bucket.",
"fill": 10,
"id": 12,
"id": 15,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -1035,7 +1278,7 @@
"datasource": "$datasource",
"description": "Shows how long has it taken to execute operations against the bucket, in quantiles.",
"fill": 1,
"id": 13,
"id": 16,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -1139,7 +1382,7 @@
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"id": 14,
"id": 17,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -1255,7 +1498,7 @@
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"id": 15,
"id": 18,
"legend": {
"avg": false,
"current": false,
Expand Down Expand Up @@ -1331,7 +1574,7 @@
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"id": 16,
"id": 19,
"legend": {
"avg": false,
"current": false,
Expand Down
34 changes: 34 additions & 0 deletions mixin/dashboards/compact.libsonnet
View file
Open in desktop
Expand Up @@ -80,6 +80,40 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet';
g.latencyPanel('thanos_compact_garbage_collection_duration_seconds', 'namespace="$namespace",job=~"$job"')
)
)
.addRow(
g.row('Blocks deletion')
.addPanel(
g.panel(
'Deletion Rate',
'Shows deletion rate of blocks already marked for deletion.'
) +
g.queryPanel(
'sum(rate(thanos_compact_blocks_cleaned_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)',
'Blocks cleanup {{job}}'
) +
g.stack
)
.addPanel(
g.panel(
'Deletion Error Rate',
'Shows deletion failures rate of blocks already marked for deletion.'
) +
g.queryPanel(
'sum(rate(thanos_compact_block_cleanup_failures_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)',
'Blocks cleanup failures {{job}}'
)
)
.addPanel(
g.panel(
'Marking Rate',
'Shows rate at which blocks are marked for deletion (from GC and retention policy).'
) +
g.queryPanel(
'sum(rate(thanos_compact_blocks_marked_for_deletion_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)',
'Blocks marked {{job}}'
)
)
)
.addRow(
g.row('Sync Meta')
.addPanel(
Expand Down