From c33b87cafb45f287f16b081fe865340e6260f6ce Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 22 Jan 2022 00:59:05 +0100 Subject: [PATCH] Improve compatibility with Grafana 8.3 / dashboard schema version 33 --- CHANGES.rst | 1 + doc/backlog.rst | 6 +++ grafana_wtf/core.py | 81 ++++++++++++++++++++++++----------- grafana_wtf/model.py | 20 +++++++++ grafana_wtf/tabular_report.py | 43 ++++++++++++++----- tests/test_commands.py | 29 +++++++------ 6 files changed, 130 insertions(+), 50 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b569664..dcc8d06 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,7 @@ in progress - Add two more examples about using `explore dashboards` with `jq` - CI: Prepare test suite for testing two different dashboard schema versions, v27 and v33 - Improve determinism by returning stable sort order of dashboard results +- Improve compatibility with Grafana 8.3 by handling dashboard schema version 33 properly 2021-12-11 0.12.0 ================= diff --git a/doc/backlog.rst b/doc/backlog.rst index c37b199..4a76b53 100644 --- a/doc/backlog.rst +++ b/doc/backlog.rst @@ -6,6 +6,12 @@ grafana-wtf backlog ****** Prio 1 ****** +- [o] With Grafana >8.3, resolve datasource name and add to ``{'type': 'influxdb', 'uid': 'PDF2762CDFF14A314'}`` + + +********* +Prio 1.25 +********* - [o] Statistics reports for data sources and panels: https://github.com/panodata/grafana-wtf/issues/18 - [o] Finding invalid data sources: https://github.com/panodata/grafana-wtf/issues/19 - [o] Add test fixture for adding dashboards at runtime from branch ``amo/test-dashboard-runtime`` diff --git a/grafana_wtf/core.py b/grafana_wtf/core.py index a8dcb25..e4a827c 100644 --- a/grafana_wtf/core.py +++ b/grafana_wtf/core.py @@ -2,6 +2,7 @@ # (c) 2019-2021 Andreas Motl # License: GNU Affero General Public License, Version 3 import asyncio +import dataclasses import json import logging from collections import OrderedDict @@ -18,6 +19,7 @@ DashboardDetails, DashboardExplorationItem, DatasourceExplorationItem, + DatasourceItem, GrafanaDataModel, ) from grafana_wtf.monkey import monkeypatch_grafana_api @@ -390,9 +392,10 @@ def explore_datasources(self): # Compute list of exploration items, associating datasources with the dashboards that use them. results_used = [] results_unused = [] - for name in sorted(ix.datasource_by_name): - datasource = ix.datasource_by_name[name] - dashboard_uids = ix.datasource_dashboard_index.get(name, []) + for ds_identifier in sorted(ix.datasource_by_ident): + + datasource = ix.datasource_by_ident[ds_identifier] + dashboard_uids = ix.datasource_dashboard_index.get(ds_identifier, []) dashboards = list(map(ix.dashboard_by_uid.get, dashboard_uids)) item = DatasourceExplorationItem(datasource=datasource, used_in=dashboards, grafana_url=self.grafana_url) @@ -404,6 +407,9 @@ def explore_datasources(self): else: results_unused.append(result) + results_used = sorted(results_used, key=lambda x: x["datasource"]["uid"] or x["datasource"]["name"]) + results_unused = sorted(results_unused, key=lambda x: x["datasource"]["uid"] or x["datasource"]["name"]) + response = OrderedDict( used=results_used, unused=results_unused, @@ -422,18 +428,20 @@ def explore_dashboards(self): for uid in sorted(ix.dashboard_by_uid): dashboard = ix.dashboard_by_uid[uid] - datasource_names = ix.dashboard_datasource_index[uid] + datasource_items = ix.dashboard_datasource_index[uid] datasources_existing = [] - datasource_names_missing = [] - for datasource_name in datasource_names: - if datasource_name == "-- Grafana --": + datasources_missing = [] + for datasource_item in datasource_items: + if datasource_item.name == "-- Grafana --": continue - datasource = ix.datasource_by_name.get(datasource_name) + datasource_by_uid = ix.datasource_by_uid.get(datasource_item.uid) + datasource_by_name = ix.datasource_by_name.get(datasource_item.name) + datasource = datasource_by_uid or datasource_by_name if datasource: datasources_existing.append(datasource) else: - datasource_names_missing.append({"name": datasource_name}) + datasources_missing.append(dataclasses.asdict(datasource_item)) item = DashboardExplorationItem( dashboard=dashboard, datasources=datasources_existing, grafana_url=self.grafana_url ) @@ -442,8 +450,8 @@ def explore_dashboards(self): result = item.format_compact() # Add information about missing data sources. - if datasource_names_missing: - result["datasources_missing"] = datasource_names_missing + if datasources_missing: + result["datasources_missing"] = datasources_missing results.append(result) @@ -456,6 +464,8 @@ def __init__(self, engine: GrafanaWtf): # Prepare index data structures. self.dashboard_by_uid = {} + self.datasource_by_ident = {} + self.datasource_by_uid = {} self.datasource_by_name = {} self.dashboard_datasource_index = {} self.datasource_dashboard_index = {} @@ -472,12 +482,16 @@ def index(self): self.index_datasources() @staticmethod - def collect_datasource_names(element): - names = [] + def collect_datasource_items(element): + items = [] for node in element: if "datasource" in node and node["datasource"]: - names.append(node.datasource) - return list(sorted(set(names))) + ds = node.datasource + if isinstance(ds, Munch): + ds = dict(ds) + if ds not in items: + items.append(ds) + return sorted(items) def index_dashboards(self): @@ -496,21 +510,36 @@ def index_dashboards(self): self.dashboard_by_uid[uid] = dashboard # Map to data source names. - ds_panels = self.collect_datasource_names(dbdetails.panels) - ds_annotations = self.collect_datasource_names(dbdetails.annotations) - ds_templating = self.collect_datasource_names(dbdetails.templating) - self.dashboard_datasource_index[uid] = list(sorted(set(ds_panels + ds_annotations + ds_templating))) + ds_panels = self.collect_datasource_items(dbdetails.panels) + ds_annotations = self.collect_datasource_items(dbdetails.annotations) + ds_templating = self.collect_datasource_items(dbdetails.templating) + + results = [] + for bucket in ds_panels, ds_annotations, ds_templating: + for item in bucket: + item = DatasourceItem.from_payload(item) + if item not in results: + results.append(item) + self.dashboard_datasource_index[uid] = results def index_datasources(self): + self.datasource_by_ident = {} + self.datasource_by_uid = {} self.datasource_by_name = {} self.datasource_dashboard_index = {} for datasource in self.datasources: - name = datasource.name - self.datasource_by_name[name] = datasource - - for dashboard_uid, datasource_names in self.dashboard_datasource_index.items(): - for datasource_name in datasource_names: - self.datasource_dashboard_index.setdefault(datasource_name, []) - self.datasource_dashboard_index[datasource_name].append(dashboard_uid) + datasource_name_or_uid = datasource.uid or datasource.name + self.datasource_by_ident[datasource_name_or_uid] = datasource + self.datasource_by_uid[datasource.uid] = datasource + self.datasource_by_name[datasource.name] = datasource + + for dashboard_uid, datasource_items in self.dashboard_datasource_index.items(): + datasource_item: DatasourceItem + for datasource_item in datasource_items: + datasource_name_or_uid = datasource_item.uid or datasource_item.name + if datasource_name_or_uid in self.datasource_by_name: + datasource_name_or_uid = self.datasource_by_name[datasource_name_or_uid].uid + self.datasource_dashboard_index.setdefault(datasource_name_or_uid, []) + self.datasource_dashboard_index[datasource_name_or_uid].append(dashboard_uid) diff --git a/grafana_wtf/model.py b/grafana_wtf/model.py index fc9035f..ab7a85d 100644 --- a/grafana_wtf/model.py +++ b/grafana_wtf/model.py @@ -42,6 +42,24 @@ def templating(self) -> List: return self.dashboard.dashboard.get("templating", {}).get("list", []) +@dataclasses.dataclass +class DatasourceItem: + uid: Optional[str] = None + name: Optional[str] = None + type: Optional[str] = None + url: Optional[str] = None + + @classmethod + def from_payload(cls, payload: any): + if isinstance(payload, Munch): + payload = dict(payload) + if isinstance(payload, dict): + return cls(**payload) + if isinstance(payload, str): + return cls(name=payload) + raise TypeError(f"Unknown payload type for DatasourceItem: {type(payload)}") + + @dataclasses.dataclass class DatasourceExplorationItem: datasource: Munch @@ -50,6 +68,7 @@ class DatasourceExplorationItem: def format_compact(self): dsshort = OrderedDict( + uid=self.datasource.uid, name=self.datasource.name, type=self.datasource.type, url=self.datasource.url, @@ -84,6 +103,7 @@ def format_compact(self): for datasource in self.datasources: item.setdefault("datasources", []) dsshort = OrderedDict( + uid=datasource.uid, name=datasource.name, type=datasource.type, url=datasource.url, diff --git a/grafana_wtf/tabular_report.py b/grafana_wtf/tabular_report.py index e4e8230..46e5410 100644 --- a/grafana_wtf/tabular_report.py +++ b/grafana_wtf/tabular_report.py @@ -2,6 +2,7 @@ from collections import OrderedDict from jsonpath_rw import parse +from munch import Munch from tabulate import tabulate from grafana_wtf.report import WtfReport @@ -15,8 +16,8 @@ def __init__(self, grafana_url, tblfmt="psql", verbose=False): def output_items(self, label, items, url_callback): items_rows = [ { - "type": label, - "name": self.get_item_name(item), + "Type": label, + "Name": self.get_item_name(item), **self.get_bibdata_dict(item, URL=url_callback(item)), } for item in items @@ -32,15 +33,35 @@ def get_bibdata_dict(self, item, **kwargs): bibdata["Title"] = item.data.dashboard.title bibdata["Folder"] = item.data.meta.folderTitle bibdata["UID"] = item.data.dashboard.uid - bibdata["Creation date"] = f"{item.data.meta.created}" - bibdata["created by"] = item.data.meta.createdBy - bibdata["last update date"] = f"{item.data.meta.updated}" + bibdata["Created"] = f"{item.data.meta.created}" + bibdata["Updated"] = f"{item.data.meta.updated}" + bibdata["Created by"] = item.data.meta.createdBy + + # FIXME: The test fixtures are currently not deterministic, + # because Grafana is not cleared on each test case. if "PYTEST_CURRENT_TEST" not in os.environ: - bibdata["updated by"] = item.data.meta.updatedBy - _finder = parse("$..datasource") - _datasources = _finder.find(item) - bibdata["datasources"] = ",".join( - sorted(set([str(_ds.value) for _ds in _datasources if _ds.value])) if _datasources else "" - ) + bibdata["Updated by"] = item.data.meta.updatedBy + + bibdata["Datasources"] = ",".join(map(str, self.get_datasources(item))) bibdata.update(kwargs) return bibdata + + def get_datasources(self, item): + + # Query datasources. + _finder = parse("$..datasource") + _datasources = _finder.find(item) + + # Compute unique list of datasources. + datasources = [] + for _ds in _datasources: + if not _ds.value: + continue + if isinstance(_ds.value, Munch): + value = dict(_ds.value) + else: + value = str(_ds.value) + if value not in datasources: + datasources.append(value) + + return datasources diff --git a/tests/test_commands.py b/tests/test_commands.py index f35d0d1..213b536 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -67,7 +67,7 @@ def test_find_textual_dashboard_success(docker_grafana, capsys): assert "dashboard.panels.[7].panels.[0].targets.[0].measurement: ldi_readings" in captured.out -def test_find_textual_datasource_dashboard_success(docker_grafana, capsys): +def test_find_textual_datasource_success(docker_grafana, capsys): set_command("find ldi_v2") grafana_wtf.commands.run() captured = capsys.readouterr() @@ -77,7 +77,7 @@ def test_find_textual_datasource_dashboard_success(docker_grafana, capsys): assert "name: ldi_v2" in captured.out assert "database: ldi_v2" in captured.out - assert "Dashboards: 2 hits" in captured.out + assert "Dashboards: 1 hits" in captured.out assert "luftdaten-info-generic-trend" in captured.out assert "dashboard.panels.[1].datasource: ldi_v2" in captured.out assert "dashboard.panels.[7].panels.[0].datasource: ldi_v2" in captured.out @@ -91,13 +91,13 @@ def test_find_tabular_dashboard_success(docker_grafana, capsys): assert 'Searching for expression "ldi_readings" at Grafana instance http://localhost:3000' in captured.out reference_table = """ -| type | name | Title | Folder | UID | Creation date | created by | last update date | datasources | URL | -|:-----------|:---------------------------------|:---------------------------------|:----------|:----------|:---------------------|:-------------|:---------------------|:---------------------------------|:-------------------------------------------------------------------| -| Dashboards | luftdaten-info-generic-trend-v27 | luftdaten.info generic trend v27 | Testdrive | ioUrPwQiz | xxxx-xx-xxTxx:xx:xxZ | Anonymous | xxxx-xx-xxTxx:xx:xxZ | -- Grafana --,ldi_v2,weatherbase | http://localhost:3000/d/ioUrPwQiz/luftdaten-info-generic-trend-v27 | -| Dashboards | luftdaten-info-generic-trend-v33 | luftdaten.info generic trend v33 | Testdrive | jpVsQxRja | xxxx-xx-xxTxx:xx:xxZ | Anonymous | xxxx-xx-xxTxx:xx:xxZ | -- Grafana --,ldi_v2,weatherbase | http://localhost:3000/d/jpVsQxRja/luftdaten-info-generic-trend-v33 | +| Type | Name | Title | Folder | UID | Created | Updated | Created by | Datasources | URL | +|:-----------|:---------------------------------|:---------------------------------|:----------|:----------|:---------------------|:---------------------|:-------------|:--------------------------------------------------------------------------------------|:-------------------------------------------------------------------| +| Dashboards | luftdaten-info-generic-trend-v27 | luftdaten.info generic trend v27 | Testdrive | ioUrPwQiz | xxxx-xx-xxTxx:xx:xxZ | xxxx-xx-xxTxx:xx:xxZ | Anonymous | -- Grafana --,ldi_v2,weatherbase | http://localhost:3000/d/ioUrPwQiz/luftdaten-info-generic-trend-v27 | +| Dashboards | luftdaten-info-generic-trend-v33 | luftdaten.info generic trend v33 | Testdrive | jpVsQxRja | xxxx-xx-xxTxx:xx:xxZ | xxxx-xx-xxTxx:xx:xxZ | Anonymous | -- Grafana --,{'type': 'influxdb', 'uid': 'PDF2762CDFF14A314'},{'uid': 'weatherbase'} | http://localhost:3000/d/jpVsQxRja/luftdaten-info-generic-trend-v33 | """.strip() - output_table = captured.out[captured.out.find("| type") :] + output_table = captured.out[captured.out.find("| Type") :] output_table_normalized = re.sub( r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", r"xxxx-xx-xxTxx:xx:xxZ", output_table ).strip() @@ -123,8 +123,8 @@ def test_replace_dashboard_success(docker_grafana, capsys): # assert "name: ldi_v2" in captured.out # assert "database: ldi_v2" in captured.out - assert "Dashboards: 2 hits" in captured.out - assert "luftdaten-info-generic-trend" in captured.out + assert "Dashboards: 1 hits" in captured.out + assert "luftdaten-info-generic-trend-v27" in captured.out assert "Folder Testdrive" in captured.out assert "dashboard.panels.[1].datasource: ldi_v3" in captured.out assert "dashboard.panels.[7].panels.[0].datasource: ldi_v3" in captured.out @@ -222,15 +222,18 @@ def test_explore_dashboards(docker_grafana, create_datasource, capsys, caplog): assert len(data) >= 1 missing = find_all_missing_datasources(data) - assert missing == ["weatherbase"] + + # Those are bogus! + assert missing[0]["name"] == "weatherbase" + assert missing[1]["uid"] == "weatherbase" def find_all_missing_datasources(data): - missing_names = [] + missing_items = [] for item in data: if "datasources_missing" in item: - missing_names += map(operator.itemgetter("name"), item["datasources_missing"]) - return sorted(set(missing_names)) + missing_items += item["datasources_missing"] + return sorted(missing_items, key=lambda x: x["uid"] or x["name"]) def test_info(docker_grafana, capsys, caplog):