MixedDatasource: stream results and keep track each sub-request

[ryantxu]

This PR changes MixedDatasource so that if all requests are pointing to the same datasource, it will incrementally stream back results as they come in.

If the queries are really mixed (ie multiple datasources) they will behave identical to before.

Here is a test dashboard:

dashboard.json

{
  "__inputs": [
    {
      "name": "DS_GDEV-TESTDATA",
      "label": "gdev-testdata",
      "description": "",
      "type": "datasource",
      "pluginId": "testdata",
      "pluginName": "TestData DB"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "6.4.0-pre"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "testdata",
      "name": "TestData DB",
      "version": "1.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "panels": [
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "-- Mixed --",
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "id": 2,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "1s",
          "datasource": "${DS_GDEV-TESTDATA}",
          "refId": "A",
          "scenarioId": "slow_query",
          "stringInput": "1s"
        },
        {
          "alias": "2s",
          "datasource": "${DS_GDEV-TESTDATA}",
          "refId": "B",
          "scenarioId": "slow_query",
          "stringInput": "2s"
        },
        {
          "alias": "5s",
          "datasource": "${DS_GDEV-TESTDATA}",
          "refId": "C",
          "scenarioId": "slow_query",
          "stringInput": "5s"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Slow Queries w/ streaming results (mixed)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_GDEV-TESTDATA}",
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "id": 3,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "1s",
          "refId": "A",
          "scenarioId": "slow_query",
          "stringInput": "1s"
        },
        {
          "alias": "2s",
          "refId": "B",
          "scenarioId": "slow_query",
          "stringInput": "2s"
        },
        {
          "alias": "5s",
          "refId": "C",
          "scenarioId": "slow_query",
          "stringInput": "5s"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Slow queries (no streaming)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "schemaVersion": 19,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "mixed slow queries",
  "uid": "qVmff8dWk",
  "version": 6
}

[torkelo]

Looking really good! not sure the throttling is working 100%, with 5 queries that return 1 series each, and viz repeated gauges I get first a single no data render, then one series rendering, then the rest, so is a bit flickery.

[hugohaggmark]

Without digging very deep into this, I think this could replace the current Prometheus delta implementation in Explore. Great work!

[ryantxu]

Some UX issues we need to think about after this is merged:

• the loading icon when panel state is streaming... maybe a solid circle? green circle?
• when some series are loading in the graph, it would be cool to have the loading state in the legend. This is maybe something we could think about after POC: Testing using new legend in old graph #17970

[aocenas]

So this is one of the thing why I do not like this API very much. The behaviour of this function changes drastically based on whether there are multiple datasources or just one, which you as a caller do not see until you go into the params. This means you would need to always expect both promise and a stream just to be sure which then makes observer non optional for most cases which we cannot put in the type because then it would break backward compatibility. Right now this feels a bit like if the interface was callFunction('functionName') and you need to know internal implementation of that call to know what it will do and how to handle that.

[hugohaggmark]

Would it make more sense if the api looked like this

async query(request: DataQueryRequest<DataQuery>): Observable<DataQueryResponse> 

This way you would always expect a stream of data, either from a single request or from multiple requests or live streaming.

[hugohaggmark]

Or do we want to avoid the RxJs dependency?

[torkelo]

I think we tried something similar in #16714

Using the old (v3-v6) streaming support was handled by returning a Subject. But think it's tricky to just return an observable of DataQueryResponse, think it would have to be more of a Observable of DataStreamState or something inspired by that or expand DataQueryResponse with all the things needed to track streaming events (key & source request).

[aocenas]

Well I think right now with the observer callback we already push the data there as an events with key for the source request if I am not mistaken. We could still be returning that in one observable, or possibly returning Observable of Observables, with one stream per source request that you can flatten if needed. Something like Observable<Observable<DataQueryResponse> | DataQueryResponse> maybe, though you would need to assert the type on the way back based on what you sent in which I am not that fond of.

[hugohaggmark]

Good points, I think that having more meta data like ((key & source request) and DataQueryResponse would be a simpler solution. The only thing I would be interested in as a subscriber of the stream are the actual "address" of the data not if there are one or multiple streams or if it's live streaming.

[aocenas]

How do we know we are done here? What if the datasource returns a promise and keeps pushing data through the observer?

[ryantxu]

in the panelQueryRunner it checks that the events came from the last request. if not, it unsubscribes.

panelQueryRunner reports done when the promise and all stream keys report they are done

[ryantxu]

I like the idea of trying something like

async observe(request: DataQueryRequest<DataQuery>): Observable<DataStreamState>

note

• it would return the state and not the direct response
• the function needs to be have a different name than query since that is taken and needs to exist for backwards compatibility

BUT -- I think improving the DataSourceAPI streaming can and should be a different issue than this one. The bulk of this is figuring out how we would identify subrequest in the state and updating the UI to reflect if they are running.

[aocenas]

Are we actually modifying the request we got as a param here? To signal something to the caller? If that is the case, I really, really don't like that. Also few questions for the subrequests:

• Is this a one level of subrequests? If so is that some arbitrary limit or just because of how mixed DS works?
• Why subrequests here at all? I would kinda expect the grouping of queries be transparent to the caller. Something along the lines of getting queries, group by DS so they can be sent together so DS can do any optimisation. Ungroup them on the way out so that caller does not need to know about the grouping at all.

[ryantxu]

If that is the case, I really, really don't like that.

Good point -- I will find some other way

Is this a one level of subrequests? If so is that some arbitrary limit or just because of how mixed DS works?

It is not limited. subRequests can also make subRequests, the only constraint is that the requestId for sub-requests needs to start with the id of its parent.

Why subrequests here at all?

What you describe is exactly the behavior you get when there are multiple datasources selected -- if you are actually using mixed queries! Sub-requests are only triggered if you use mixed datasource, but everything points to a single datasource.

I agree the semantics may be akward -- but not terribly so. My goal is using explicit sub-requets in external datasources, but we need something in core to be able to clearly see and test this behavior. It is already used in Prometheus, but there is no status reporting.

[aocenas]

What you describe is exactly the behavior you get when there are multiple ...

My main point is more about how this is leaked to the caller. Like caller probably does not need to know this? I think getting a different streams with unique ID is enough and it does not need to know about whether they are hierarchical in some way. Especially that from the UI perspective there isn't really a hierarchy in the queries.

So if this is only internal structuring it is fine (also the ID scheme is fine), but we do request.subRequests.push(sub); later on. So this seems again like a mutation of param to signal some info upstream. So apart from problem with param mutation, I think caller does not really need that information anyway. Or is there an usecase when it is necessary?

[torkelo]

I agree with this, think leaking the concept of subRequests might be unnecessary. And could be an internal detail in the mixed data source. Question is how to communicate the state of individual queries sent in the request to the data source. Maybe we can do this with the refId that is part of the DataFrame result?

[aocenas]

There is DataStreamState.key property so I assume that is the unique identifier of stream for the consumer? query.refId is the one that is passed to it here.

[ryantxu]

closing this because we have a good solution now