InfluxDB: Parse data for table view to have parity with frontend parser #78365
Conversation
|
Hello @itsmylife!
Please, if the current pull request addresses a bug fix, label it with the |
gtk-grafana
requested review from
gtk-grafana and
kylebrandt
and removed request for
gtk-grafana
tolzhabayev
added
the
product-approved
| return &v | ||
| } | ||
|
|
||
| const tableResultFormatInfluxResponse1 = `{ |
There was a problem hiding this comment.
It would be better to use the pattern we use in prom here for testing with the json test_data files. I don't think it is a blocker on the PR, but we should probably make an issue to update the tests in this DS.
There was a problem hiding this comment.
That's a good call. I will also refactor the time_series parsing since we have a lot of duplicate code with this PR. When I do that, I will refactor testing too. I'll create an issue for myself to track this.
…ntend parser (#78551) InfluxDB: Parse data for table view to have parity with frontend parser (#78365) * Use TimeSeriesWide format for table response * fix group by query result parsing * handle labels * provide a test where result has no tags * parsing results without time column * clean the code * remove the comment line * more cleaning * lint (cherry picked from commit c5f3ce1) Co-authored-by: ismail simsek <ismailsimsek09@gmail.com>
…er (#78365) * Use TimeSeriesWide format for table response * fix group by query result parsing * handle labels * provide a test where result has no tags * parsing results without time column * clean the code * remove the comment line * more cleaning * lint
What is this feature?
InfluxDB has a frontend response parser which will be not used as we are moving towards backend migration. This means the query will be fetched and parsed on the backend and will return as
DataFrame.The current data format is compatible with
timeseriesbut not with thetableview %100.Users have to use
transformationsand other techniques to reach to same view as they had before.This is a tedious process to go through. This PR is resolving that by adding a separate parsing logic for table view.
Part of #65045
Frontend Mode
Backend Mode before PR
Backend Mode after PR
Why do we need this feature?
To have parity with frontend parser and no breaking changes while having
influxdbBackendMigrationfeature toggle enabled.Who is this feature for?
InfluxDB
influxqlusers who have table views in their dashboardsHow to test
dashboard.json for testing
{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 8, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "id": 8, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "measurement": "cpu", "orderByTime": "ASC", "policy": "default", "query": "SHOW MEASUREMENTS", "rawQuery": true, "refId": "A", "resultFormat": "table", "select": [ [ { "params": [ "value" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [] } ], "title": "show measurements", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "id": 7, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "query": "SHOW RETENTION POLICIES on \"mybucket\"", "rawQuery": true, "refId": "A", "resultFormat": "table" } ], "title": "show retention policies", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "id": 5, "interval": "20m", "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "alias": "$tag_cpu", "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "cpu::tag" ], "type": "tag" }, { "params": [ "null" ], "type": "fill" } ], "measurement": "cpu", "orderByTime": "ASC", "policy": "default", "query": "SELECT mean(\"usage_idle\") FROM \"default\".\"cpu\" WHERE (\"host\"::tag = '41c9dcf3b960') AND $timeFilter GROUP BY time($__interval), \"cpu\"::tag fill(null)", "rawQuery": true, "refId": "A", "resultFormat": "table", "select": [ [ { "params": [ "usage_idle" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [ { "key": "host::tag", "operator": "=", "value": "41c9dcf3b960" } ] } ], "title": "select mean where host and group by cpu tag", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "id": 6, "interval": "20m", "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "measurement": "cpu", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "table", "select": [ [ { "params": [ "usage_idle" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [] } ], "title": "no tags", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, "id": 4, "interval": "1m", "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "alias": "fsdsfdsfdsdfsfd", "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "measurement": "processes", "orderByTime": "ASC", "policy": "default", "query": "SELECT * FROM \"default\".\"cpu\" WHERE (\"host\"::tag = '41c9dcf3b960') AND $timeFilter", "rawQuery": true, "refId": "A", "resultFormat": "table", "select": [ [ { "params": [ "load15" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [ { "key": "host::tag", "operator": "=", "value": "41c9dcf3b960" } ] } ], "title": "select * from cpu where tag is something", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 24 }, "id": 3, "interval": "1m", "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "measurement": "diskio", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "table", "select": [ [ { "params": [ "*" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [ { "key": "name::tag", "operator": "=", "value": "vda" } ] } ], "title": "select * with mean", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, "id": 1, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [], "hide": false, "measurement": "cpu", "orderByTime": "ASC", "policy": "default", "query": "SELECT mean(\"usage_idle\"), min(\"usage_idle\"), PERCENTILE(\"usage_idle\", 90) as p90, PERCENTILE(\"usage_idle\", 95) as p95, max(\"usage_idle\") as max FROM \"cpu\" WHERE (\"host\"::tag = '41c9dcf3b960') and $timeFilter GROUP BY \"cpu\" fill(null)", "rawQuery": true, "refId": "B", "resultFormat": "table", "select": [ [ { "params": [ "usage_idle" ], "type": "field" } ], [ { "params": [ "usage_iowait" ], "type": "field" } ] ], "tags": [] } ], "title": "select with where host and group by cpu (rawQuery)", "type": "table" }, { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, "id": 2, "interval": "1m", "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "9.5.13", "targets": [ { "datasource": { "type": "influxdb", "uid": "a712866b-d847-499b-9875-72c06f0eceeb" }, "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "measurement": "cpu", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "table", "select": [ [ { "params": [ "usage_idle" ], "type": "field" }, { "params": [], "type": "mean" }, { "params": [ "idle_mean" ], "type": "alias" } ], [ { "params": [ "usage_iowait" ], "type": "field" }, { "params": [], "type": "mean" }, { "params": [ "iowait_mean" ], "type": "alias" } ] ], "tags": [] } ], "title": "two select with alias group by time fill null", "type": "table" } ], "refresh": "", "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "2023-11-15T11:00:00.000Z", "to": "2023-11-15T12:00:00.000Z" }, "timepicker": {}, "timezone": "", "title": "influx table investigation with frontend parser", "uid": "a73e531a-e843-498a-a9e3-29f230a91ee6", "version": 14, "weekStart": "" }influxdbBackendMigrationfeature toggle falseinfluxqlwhich has table.influxdbBackendMigrationfeature togglePlease check that: