diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py index 498f78ad2c..4c22ea2fe7 100644 --- a/dvc/commands/plots.py +++ b/dvc/commands/plots.py @@ -59,49 +59,6 @@ def _show_json( ui.write_json(compact({"errors": all_errors, "data": data}), highlight=False) -def _adjust_vega_renderers(renderers): - from dvc.render import REVISION_FIELD, VERSION_FIELD - from dvc_render import VegaRenderer - - for r in renderers: - if isinstance(r, VegaRenderer): - if _data_versions_count(r) > 1: - summary = _summarize_version_infos(r) - for dp in r.datapoints: - vi = dp.pop(VERSION_FIELD, {}) - keys = list(vi.keys()) - for key in keys: - if not (len(summary.get(key, set())) > 1): - vi.pop(key) - if vi: - dp["rev"] = "::".join(vi.values()) - else: - for dp in r.datapoints: - dp[REVISION_FIELD] = dp[VERSION_FIELD]["revision"] - dp.pop(VERSION_FIELD, {}) - - -def _summarize_version_infos(renderer): - from collections import defaultdict - - from dvc.render import VERSION_FIELD - - result = defaultdict(set) - - for dp in renderer.datapoints: - for key, value in dp.get(VERSION_FIELD, {}).items(): - result[key].add(value) - return dict(result) - - -def _data_versions_count(renderer): - from itertools import product - - summary = _summarize_version_infos(renderer) - x = product(summary.get("filename", {None}), summary.get("field", {None})) - return len(set(x)) - - class CmdPlots(CmdBase): def _func(self, *args, **kwargs): raise NotImplementedError @@ -175,11 +132,10 @@ def run(self) -> int: # noqa: C901, PLR0911, PLR0912 return 0 renderers = [r.renderer for r in renderers_with_errors] - _adjust_vega_renderers(renderers) if self.args.show_vega: renderer = first(filter(lambda r: r.TYPE == "vega", renderers)) if renderer: - ui.write_json(renderer.get_filled_template(as_string=False)) + ui.write_json(renderer.get_filled_template()) return 0 output_file: Path = (Path.cwd() / out).resolve() / "index.html" diff --git a/dvc/render/__init__.py b/dvc/render/__init__.py index bedc1414cf..a3ee972fb9 100644 --- a/dvc/render/__init__.py +++ b/dvc/render/__init__.py @@ -1,7 +1,8 @@ -INDEX_FIELD = "step" -REVISION_FIELD = "rev" -FILENAME_FIELD = "filename" -VERSION_FIELD = "dvc_data_version_info" -REVISIONS_KEY = "revisions" +INDEX = "step" +REVISION = "rev" +FILENAME = "filename" +FIELD = "field" +REVISIONS = "revisions" +ANCHOR_DEFINITIONS = "anchor_definitions" TYPE_KEY = "type" -SRC_FIELD = "src" +SRC = "src" diff --git a/dvc/render/convert.py b/dvc/render/convert.py index 1d576fc3ab..bfbc6fa40e 100644 --- a/dvc/render/convert.py +++ b/dvc/render/convert.py @@ -1,7 +1,6 @@ -from collections import defaultdict from typing import Dict, List, Union -from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD +from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY from dvc.render.converter.image import ImageConverter from dvc.render.converter.vega import VegaConverter @@ -19,39 +18,31 @@ def _get_converter( raise ValueError(f"Invalid renderer class {renderer_class}") -def _group_by_rev(datapoints): - grouped = defaultdict(list) - for datapoint in datapoints: - rev = datapoint.get(VERSION_FIELD, {}).get("revision") - grouped[rev].append(datapoint) - return dict(grouped) - - def to_json(renderer, split: bool = False) -> List[Dict]: if renderer.TYPE == "vega": - grouped = _group_by_rev(renderer.datapoints) + if not renderer.datapoints: + return [] + revs = renderer.get_revs() if split: - content = renderer.get_filled_template( - skip_anchors=["data"], as_string=False - ) + content, split_content = renderer.get_partial_filled_template() else: - content = renderer.get_filled_template(as_string=False) - if grouped: - return [ - { - TYPE_KEY: renderer.TYPE, - REVISIONS_KEY: sorted(grouped.keys()), - "content": content, - "datapoints": grouped, - } - ] - return [] + content = renderer.get_filled_template() + split_content = {} + + return [ + { + TYPE_KEY: renderer.TYPE, + REVISIONS: revs, + "content": content, + **split_content, + } + ] if renderer.TYPE == "image": return [ { TYPE_KEY: renderer.TYPE, - REVISIONS_KEY: [datapoint.get(REVISION_FIELD)], - "url": datapoint.get(SRC_FIELD), + REVISIONS: [datapoint.get(REVISION)], + "url": datapoint.get(SRC), } for datapoint in renderer.datapoints ] diff --git a/dvc/render/converter/image.py b/dvc/render/converter/image.py index fba390b306..fed3a02f4c 100644 --- a/dvc/render/converter/image.py +++ b/dvc/render/converter/image.py @@ -2,7 +2,7 @@ import os from typing import TYPE_CHECKING, Any, Dict, List, Tuple -from dvc.render import FILENAME_FIELD, REVISION_FIELD, SRC_FIELD +from dvc.render import FILENAME, REVISION, SRC from . import Converter @@ -58,9 +58,9 @@ def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]: else: src = self._encode_image(image_data) datapoint = { - REVISION_FIELD: revision, - FILENAME_FIELD: filename, - SRC_FIELD: src, + REVISION: revision, + FILENAME: filename, + SRC: src, } datapoints.append(datapoint) return datapoints, properties diff --git a/dvc/render/converter/vega.py b/dvc/render/converter/vega.py index 6b45be7131..b5cb5a3efd 100644 --- a/dvc/render/converter/vega.py +++ b/dvc/render/converter/vega.py @@ -4,7 +4,7 @@ from funcy import first, last from dvc.exceptions import DvcException -from dvc.render import FILENAME_FIELD, INDEX_FIELD, VERSION_FIELD +from dvc.render import FIELD, FILENAME, INDEX, REVISION from . import Converter @@ -112,7 +112,6 @@ def __init__( ): super().__init__(plot_id, data, properties) self.plot_id = plot_id - self.inferred_properties: Dict = {} def _infer_y_from_data(self): if self.plot_id in self.data: @@ -120,34 +119,38 @@ def _infer_y_from_data(self): if all(isinstance(item, dict) for item in lst): datapoint = first(lst) field = last(datapoint.keys()) - self.inferred_properties["y"] = {self.plot_id: field} - break + return {self.plot_id: field} + return None def _infer_x_y(self): x = self.properties.get("x", None) y = self.properties.get("y", None) + inferred_properties: Dict = {} + # Infer x. if isinstance(x, str): - self.inferred_properties["x"] = {} + inferred_properties["x"] = {} # If multiple y files, duplicate x for each file. if isinstance(y, dict): for file, fields in y.items(): # Duplicate x for each y. if isinstance(fields, list): - self.inferred_properties["x"][file] = [x] * len(fields) + inferred_properties["x"][file] = [x] * len(fields) else: - self.inferred_properties["x"][file] = x + inferred_properties["x"][file] = x # Otherwise use plot ID as file. else: - self.inferred_properties["x"][self.plot_id] = x + inferred_properties["x"][self.plot_id] = x # Infer y. if y is None: - self._infer_y_from_data() + inferred_properties["y"] = self._infer_y_from_data() # If y files not provided, use plot ID as file. elif not isinstance(y, dict): - self.inferred_properties["y"] = {self.plot_id: y} + inferred_properties["y"] = {self.plot_id: y} + + return inferred_properties def _find_datapoints(self): result = {} @@ -182,7 +185,7 @@ def infer_x_label(properties): x = properties.get("x", None) if not isinstance(x, dict): - return INDEX_FIELD + return INDEX fields = {field for _, field in _file_field(x)} if len(fields) == 1: @@ -192,7 +195,7 @@ def infer_x_label(properties): def flat_datapoints(self, revision): # noqa: C901, PLR0912 file2datapoints, properties = self.convert() - props_update = {} + props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {} xs = list(_get_xs(properties, file2datapoints)) @@ -200,15 +203,17 @@ def flat_datapoints(self, revision): # noqa: C901, PLR0912 if not xs: x_file, x_field = ( None, - INDEX_FIELD, + INDEX, ) else: x_file, x_field = xs[0] - props_update["x"] = x_field + + num_xs = len(xs) + multiple_x_fields = num_xs > 1 and len({x[1] for x in xs}) > 1 + props_update["x"] = "dvc_inferred_x_value" if multiple_x_fields else x_field ys = list(_get_ys(properties, file2datapoints)) - num_xs = len(xs) num_ys = len(ys) if num_xs > 1 and num_xs != num_ys: raise DvcException( @@ -237,6 +242,14 @@ def flat_datapoints(self, revision): # noqa: C901, PLR0912 else: common_prefix_len = 0 + props_update["anchors_y_definitions"] = [ + { + FILENAME: _get_short_y_file(y_file, common_prefix_len), + FIELD: y_field, + } + for y_file, y_field in ys + ] + for i, (y_file, y_field) in enumerate(ys): if num_xs > 1: x_file, x_field = xs[i] @@ -249,15 +262,16 @@ def flat_datapoints(self, revision): # noqa: C901, PLR0912 source_field=y_field, ) - if x_field == INDEX_FIELD and x_file is None: - _update_from_index(datapoints, INDEX_FIELD) + if x_field == INDEX and x_file is None: + _update_from_index(datapoints, INDEX) else: x_datapoints = file2datapoints.get(x_file, []) try: _update_from_field( datapoints, - field=x_field, + field="dvc_inferred_x_value" if multiple_x_fields else x_field, source_datapoints=x_datapoints, + source_field=x_field, ) except IndexError: raise DvcException( # noqa: B904 @@ -266,15 +280,12 @@ def flat_datapoints(self, revision): # noqa: C901, PLR0912 "They have to have same length." ) - y_file_short = y_file[common_prefix_len:].strip("/\\") _update_all( datapoints, update_dict={ - VERSION_FIELD: { - "revision": revision, - FILENAME_FIELD: y_file_short, - "field": y_field, - } + REVISION: revision, + FILENAME: _get_short_y_file(y_file, common_prefix_len), + FIELD: y_field, }, ) @@ -295,10 +306,10 @@ def convert( generated datapoints and updated properties. `x`, `y` values and labels are inferred and always provided. """ - self._infer_x_y() + inferred_properties = self._infer_x_y() datapoints = self._find_datapoints() - properties = {**self.properties, **self.inferred_properties} + properties = {**self.properties, **inferred_properties} properties["y_label"] = self.infer_y_label(properties) properties["x_label"] = self.infer_x_label(properties) @@ -306,6 +317,10 @@ def convert( return datapoints, properties +def _get_short_y_file(y_file, common_prefix_len): + return y_file[common_prefix_len:].strip("/\\") + + def _update_from_field( target_datapoints: List[Dict], field: str, diff --git a/dvc/render/match.py b/dvc/render/match.py index f7fb6e7a03..0a861cb4fd 100644 --- a/dvc/render/match.py +++ b/dvc/render/match.py @@ -80,7 +80,7 @@ def match_defs_renderers( # noqa: C901, PLR0912 for plot_id, group in plots_data.group_definitions().items(): plot_datapoints: List[Dict] = [] props = _squash_plots_properties(group) - final_props: Dict = {} + first_props: Dict = {} def_errors: Dict[str, Exception] = {} src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict) @@ -90,6 +90,7 @@ def match_defs_renderers( # noqa: C901, PLR0912 if templates_dir is not None: props["template_dir"] = templates_dir + revs = [] for rev, inner_id, plot_definition in group: plot_sources = infer_data_sources(inner_id, plot_definition) definitions_data = plots_data.get_definition_data(plot_sources, rev) @@ -109,19 +110,24 @@ def match_defs_renderers( # noqa: C901, PLR0912 try: dps, rev_props = converter.flat_datapoints(rev) + if dps and rev not in revs: + revs.append(rev) except Exception as e: # noqa: BLE001 logger.warning("In %r, %s", rev, str(e).lower()) def_errors[rev] = e continue - if not final_props and rev_props: - final_props = rev_props + if not first_props and rev_props: + first_props = rev_props plot_datapoints.extend(dps) - if "title" not in final_props: - final_props["title"] = renderer_id + if "title" not in first_props: + first_props["title"] = renderer_id + + if revs: + first_props["revs_with_datapoints"] = revs if renderer_cls is not None: - renderer = renderer_cls(plot_datapoints, renderer_id, **final_props) + renderer = renderer_cls(plot_datapoints, renderer_id, **first_props) renderers.append(RendererWithErrors(renderer, dict(src_errors), def_errors)) return renderers diff --git a/pyproject.toml b/pyproject.toml index 6b397db22f..c5402434c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dynamic = ["version"] + dependencies = [ "colorama>=0.3.9", "configobj>=5.0.6", @@ -36,7 +37,7 @@ dependencies = [ "dpath<3,>=2.1.0", "dvc-data>=2.22.1,<2.23.0", "dvc-http>=2.29.0", - "dvc-render>=0.3.1,<1", + "dvc-render>=1.0.0,<2", "dvc-studio-client>=0.17.1,<1", "dvc-task>=0.3.0,<1", "flatten_dict<1,>=0.4.1", diff --git a/tests/integration/plots/test_plots.py b/tests/integration/plots/test_plots.py index 8a22918b92..4232f89ab7 100644 --- a/tests/integration/plots/test_plots.py +++ b/tests/integration/plots/test_plots.py @@ -11,7 +11,7 @@ from funcy import first from dvc.cli import main -from dvc.render import REVISION_FIELD, VERSION_FIELD +from dvc.render import ANCHOR_DEFINITIONS, FILENAME, REVISION JSON_OUT = "vis_data" @@ -102,6 +102,9 @@ def verify_vega( html_result, json_result, split_json_result, + title, + x_label, + y_label, ): if isinstance(versions, str): versions = [versions] @@ -111,27 +114,60 @@ def verify_vega( assert j[0]["type"] == "vega" assert set(j[0]["revisions"]) == set(versions) - assert json_result[0]["datapoints"] == split_json_result[0]["datapoints"] - assert set(versions) == set(json_result[0]["datapoints"].keys()) + assert ( + json_result[0]["content"]["data"]["values"] + == split_json_result[0][ANCHOR_DEFINITIONS][""] + ) + + assert set(versions) == set(json_result[0]["revisions"]) assert json_result[0]["content"]["data"]["values"] assert html_result["data"]["values"] - assert split_json_result[0]["content"]["data"]["values"] == "" - def _assert_templates_equal(html_template, filled_template, split_template): - # besides data, json and split json should be equal - path = ["data", "values"] + content_str = json.dumps(split_json_result[0]["content"]) + assert "" in content_str + assert "" in content_str + assert "" in content_str + + def _assert_templates_equal( + html_template, filled_template, split_template, title, x_label, y_label + ): + # besides split anchors, json and split json should be equal + paths = [["data", "values"], ["encoding", "color"]] tmp1 = deepcopy(html_template) tmp2 = deepcopy(filled_template) - tmp3 = deepcopy(split_template) - dpath.set(tmp1, path, {}) - dpath.set(tmp2, path, {}) - dpath.set(tmp3, path, {}) + tmp3 = json.loads( + json.dumps(split_template) + .replace('""', "300") + .replace('""', "300") + .replace("", title) + .replace("", x_label) + .replace("", y_label) + .replace( + '""', + json.dumps( + { + "name": "grid", + "select": "interval", + "bind": "scales", + } + ), + ) + ) + for path in paths: + dpath.set(tmp1, path, {}) + dpath.set(tmp2, path, {}) + dpath.set(tmp3, path, {}) assert tmp1 == tmp2 == tmp3 _assert_templates_equal( - html_result, json_result[0]["content"], split_json_result[0]["content"] + html_result, + json_result[0]["content"], + split_json_result[0]["content"], + title, + x_label, + y_label, ) @@ -193,17 +229,13 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_ ] == _update_datapoints( linear_v1, { - VERSION_FIELD: { - "revision": "workspace", - "filename": "linear.json", - "field": "y", - }, + REVISION: "workspace", }, ) assert html_result["linear.json"]["data"]["values"] == _update_datapoints( linear_v1, { - REVISION_FIELD: "workspace", + REVISION: "workspace", }, ) assert json_data["confusion.json"][0]["content"]["data"][ @@ -211,27 +243,29 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_ ] == _update_datapoints( confusion_v1, { - VERSION_FIELD: { - "revision": "workspace", - "filename": "confusion.json", - "field": "actual", - }, + REVISION: "workspace", }, ) assert html_result["confusion.json"]["data"]["values"] == _update_datapoints( confusion_v1, { - REVISION_FIELD: "workspace", + REVISION: "workspace", }, ) verify_image(tmp_dir, "workspace", "image.png", image_v1, html_path, json_data) - for plot in ["linear.json", "confusion.json"]: + for plot, title, x_label, y_label in [ + ("linear.json", "linear", "x", "y"), + ("confusion.json", "confusion matrix", "predicted", "actual"), + ]: verify_vega( "workspace", html_result[plot], json_data[plot], split_json_data[plot], + title, + x_label, + y_label, ) verify_vega_props("confusion.json", json_data, **confusion_props) @@ -250,12 +284,18 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_ verify_image(tmp_dir, "workspace", "image.png", image_v2, html_path, json_data) verify_image(tmp_dir, "HEAD", "image.png", image_v1, html_path, json_data) - for plot in ["linear.json", "confusion.json"]: + for plot, title, x_label, y_label in [ + ("linear.json", "linear", "x", "y"), + ("confusion.json", "confusion matrix", "predicted", "actual"), + ]: verify_vega( ["HEAD", "workspace"], html_result[plot], json_data[plot], split_json_data[plot], + title, + x_label, + y_label, ) verify_vega_props("confusion.json", json_data, **confusion_props) path = tmp_dir / "subdir" @@ -277,31 +317,23 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_ ] == _update_datapoints( linear_v2, { - VERSION_FIELD: { - "revision": "workspace", - "filename": "../linear.json", - "field": "y", - }, + REVISION: "workspace", }, ) + _update_datapoints( linear_v1, { - VERSION_FIELD: { - "revision": "HEAD", - "filename": "../linear.json", - "field": "y", - }, + REVISION: "HEAD", }, ) assert html_result["../linear.json"]["data"]["values"] == _update_datapoints( linear_v2, { - REVISION_FIELD: "workspace", + REVISION: "workspace", }, ) + _update_datapoints( linear_v1, { - REVISION_FIELD: "HEAD", + REVISION: "HEAD", }, ) assert json_data["../confusion.json"][0]["content"]["data"][ @@ -309,43 +341,38 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_ ] == _update_datapoints( confusion_v2, { - VERSION_FIELD: { - "revision": "workspace", - "filename": "../confusion.json", - "field": "actual", - }, + REVISION: "workspace", }, ) + _update_datapoints( confusion_v1, { - VERSION_FIELD: { - "revision": "HEAD", - "filename": "../confusion.json", - "field": "actual", - }, + REVISION: "HEAD", }, ) assert html_result["../confusion.json"]["data"]["values"] == _update_datapoints( confusion_v2, { - REVISION_FIELD: "workspace", + REVISION: "workspace", }, ) + _update_datapoints( confusion_v1, { - REVISION_FIELD: "HEAD", + REVISION: "HEAD", }, ) - for plot in [ - "../linear.json", - "../confusion.json", + for plot, title, x_label, y_label in [ + ("../linear.json", "linear", "x", "y"), + ("../confusion.json", "confusion matrix", "predicted", "actual"), ]: verify_vega( ["HEAD", "workspace"], html_result[plot], json_data[plot], split_json_data[plot], + title, + x_label, + y_label, ) verify_image( path, @@ -434,7 +461,7 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots): repo_state = repo_with_config_plots() plots = next(repo_state) - html_path, _, __ = call(capsys) + html_path, _, split_json_result = call(capsys) assert os.path.exists(html_path) html_result = extract_vega_specs( @@ -444,20 +471,28 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots): "confusion_train_vs_test", ], ) + ble = _update_datapoints( plots["data"]["linear_train.json"], { - REVISION_FIELD: "linear_train.json", + REVISION: "workspace", + FILENAME: "linear_train.json", }, ) + _update_datapoints( plots["data"]["linear_test.json"], { - REVISION_FIELD: "linear_test.json", + REVISION: "workspace", + FILENAME: "linear_test.json", }, ) assert html_result["linear_train_vs_test"]["data"]["values"] == ble - # TODO check json results once vscode is able to handle flexible plots + assert ( + split_json_result["data"]["linear_train_vs_test"][0][ANCHOR_DEFINITIONS][ + "" + ] + == ble + ) @pytest.mark.vscode @@ -466,7 +501,7 @@ def test_repo_with_dvclive_plots(tmp_dir, capsys, repo_with_dvclive_plots): for s in ("show", "diff"): _, json_result, split_json_result = call(capsys, subcommand=s) - expected_result = { + expected_result: Dict[str, Dict[str, list[str]]] = { "data": { "dvclive/plots/metrics/metric.tsv": [], }, diff --git a/tests/unit/render/test_convert.py b/tests/unit/render/test_convert.py index ea61e857d0..68db9d4db5 100644 --- a/tests/unit/render/test_convert.py +++ b/tests/unit/render/test_convert.py @@ -1,109 +1,87 @@ -from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD +import json + +import pytest + +from dvc.render import ANCHOR_DEFINITIONS, FILENAME, REVISION, REVISIONS, SRC, TYPE_KEY from dvc.render.convert import to_json def test_to_json_vega(mocker): vega_renderer = mocker.MagicMock() vega_renderer.TYPE = "vega" + vega_renderer.get_revs.return_value = ["bar", "foo"] vega_renderer.get_filled_template.return_value = {"this": "is vega"} - vega_renderer.datapoints = [ - { - "x": 1, - "y": 2, - VERSION_FIELD: {"revision": "foo"}, - "filename": "foo.json", - }, - { - "x": 2, - "y": 1, - VERSION_FIELD: {"revision": "bar"}, - "filename": "foo.json", - }, - ] result = to_json(vega_renderer) assert result[0] == { TYPE_KEY: vega_renderer.TYPE, - REVISIONS_KEY: ["bar", "foo"], + REVISIONS: ["bar", "foo"], "content": {"this": "is vega"}, - "datapoints": { - "foo": [ - { - "x": 1, - "y": 2, - "filename": "foo.json", - VERSION_FIELD: {"revision": "foo"}, - }, - ], - "bar": [ - { - "x": 2, - "y": 1, - "filename": "foo.json", - VERSION_FIELD: {"revision": "bar"}, - }, - ], - }, } vega_renderer.get_filled_template.assert_called() +@pytest.mark.vscode def test_to_json_vega_split(mocker): - vega_renderer = mocker.MagicMock() - vega_renderer.TYPE = "vega" - vega_renderer.get_filled_template.return_value = {"this": "is split vega"} - vega_renderer.datapoints = [ - { - "x": 1, - "y": 2, - VERSION_FIELD: {"revision": "foo"}, - "filename": "foo.json", - }, + revs = ["bar", "foo"] + content = json.dumps( { - "x": 2, - "y": 1, - VERSION_FIELD: {"revision": "bar"}, - "filename": "foo.json", + "this": "is split vega", + "encoding": {"color": ""}, + "data": {"values": ""}, + } + ) + anchor_definitions = { + "": { + "field": "rev", + "scale": { + "domain": revs, + "range": ["#ff0000", "#00ff00"], + }, }, - ] + "": [ + { + "x": 1, + "y": 2, + REVISION: "foo", + FILENAME: "foo.json", + }, + { + "x": 2, + "y": 1, + REVISION: "bar", + FILENAME: "foo.json", + }, + ], + } + + vega_renderer = mocker.MagicMock() + vega_renderer.TYPE = "vega" + vega_renderer.get_partial_filled_template.return_value = ( + content, + {ANCHOR_DEFINITIONS: anchor_definitions}, + ) + vega_renderer.get_revs.return_value = ["bar", "foo"] + result = to_json(vega_renderer, split=True) assert result[0] == { + ANCHOR_DEFINITIONS: anchor_definitions, TYPE_KEY: vega_renderer.TYPE, - REVISIONS_KEY: ["bar", "foo"], - "content": {"this": "is split vega"}, - "datapoints": { - "foo": [ - { - "x": 1, - "y": 2, - "filename": "foo.json", - VERSION_FIELD: {"revision": "foo"}, - } - ], - "bar": [ - { - "x": 2, - "y": 1, - "filename": "foo.json", - VERSION_FIELD: {"revision": "bar"}, - } - ], - }, + REVISIONS: revs, + "content": content, } - vega_renderer.get_filled_template.assert_called_with( - as_string=False, skip_anchors=["data"] - ) + vega_renderer.get_partial_filled_template.assert_called_once() def test_to_json_image(mocker): image_renderer = mocker.MagicMock() image_renderer.TYPE = "image" image_renderer.datapoints = [ - {SRC_FIELD: "contentfoo", REVISION_FIELD: "foo"}, - {SRC_FIELD: "contentbar", REVISION_FIELD: "bar"}, + {SRC: "contentfoo", REVISION: "foo"}, + {SRC: "contentbar", REVISION: "bar"}, ] result = to_json(image_renderer) assert result[0] == { - "url": image_renderer.datapoints[0].get(SRC_FIELD), - REVISIONS_KEY: [image_renderer.datapoints[0].get(REVISION_FIELD)], + "url": image_renderer.datapoints[0].get(SRC), + REVISIONS: [image_renderer.datapoints[0].get(REVISION)], TYPE_KEY: image_renderer.TYPE, } diff --git a/tests/unit/render/test_image_converter.py b/tests/unit/render/test_image_converter.py index 97497d8c84..20f317dbd4 100644 --- a/tests/unit/render/test_image_converter.py +++ b/tests/unit/render/test_image_converter.py @@ -1,4 +1,4 @@ -from dvc.render import REVISION_FIELD, SRC_FIELD +from dvc.render import FILENAME, REVISION, SRC from dvc.render.converter.image import ImageConverter @@ -8,9 +8,9 @@ def test_image_converter_no_out(): datapoints, _ = converter.flat_datapoints("r") assert datapoints[0] == { - REVISION_FIELD: "r", - "filename": "image.png", - SRC_FIELD: converter._encode_image(b"content"), + REVISION: "r", + FILENAME: "image.png", + SRC: converter._encode_image(b"content"), } @@ -21,9 +21,9 @@ def test_image_converter_with_out(tmp_dir): datapoints, _ = converter.flat_datapoints("r") assert datapoints[0] == { - REVISION_FIELD: "r", - "filename": "image.png", - SRC_FIELD: str(tmp_dir / "foo" / "r_image.png"), + REVISION: "r", + FILENAME: "image.png", + SRC: str(tmp_dir / "foo" / "r_image.png"), } assert (tmp_dir / "foo" / "r_image.png").read_bytes() == b"content" @@ -37,9 +37,9 @@ def test_image_converter_with_slash_in_revision(tmp_dir): datapoints, _ = converter.flat_datapoints("feature/r") assert datapoints[0] == { - REVISION_FIELD: "feature/r", - "filename": "image.png", - SRC_FIELD: str(tmp_dir / "foo" / "feature_r_image.png"), + REVISION: "feature/r", + FILENAME: "image.png", + SRC: str(tmp_dir / "foo" / "feature_r_image.png"), } assert (tmp_dir / "foo" / "feature_r_image.png").read_bytes() == b"content" diff --git a/tests/unit/render/test_match.py b/tests/unit/render/test_match.py index 58d4b5c5bb..c5f417a32b 100644 --- a/tests/unit/render/test_match.py +++ b/tests/unit/render/test_match.py @@ -1,9 +1,13 @@ import pytest from funcy import set_in -from dvc.render import VERSION_FIELD +from dvc.render import FIELD, FILENAME, REVISION from dvc.render.converter.vega import VegaConverter -from dvc.render.match import PlotsData, _squash_plots_properties, match_defs_renderers +from dvc.render.match import ( + PlotsData, + _squash_plots_properties, + match_defs_renderers, +) @pytest.mark.parametrize( @@ -158,25 +162,23 @@ def test_match_renderers(M): renderer = renderer_with_errors[0] assert renderer.datapoints == [ { - VERSION_FIELD: { - "revision": "v1", - "filename": "file.json", - "field": "y", - }, + REVISION: "v1", + FILENAME: "file.json", + FIELD: "y", "x": 1, "y": 1, }, { - VERSION_FIELD: { - "revision": "v1", - "filename": "file.json", - "field": "y", - }, + REVISION: "v1", + FILENAME: "file.json", + FIELD: "y", "x": 2, "y": 2, }, ] assert renderer.properties == { + "anchors_y_definitions": [{FILENAME: "file.json", FIELD: "y"}], + "revs_with_datapoints": ["v1"], "title": "plot_id_1", "x": "x", "y": "y", diff --git a/tests/unit/render/test_vega_converter.py b/tests/unit/render/test_vega_converter.py index 894b123098..3bdc65a66a 100644 --- a/tests/unit/render/test_vega_converter.py +++ b/tests/unit/render/test_vega_converter.py @@ -3,7 +3,7 @@ import pytest from dvc.exceptions import DvcException -from dvc.render import VERSION_FIELD +from dvc.render import FIELD, FILENAME, REVISION from dvc.render.converter.vega import FieldNotFoundError, VegaConverter, _lists @@ -35,23 +35,25 @@ def test_finding_lists(dictionary, expected_result): { "v": 1, "step": 0, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", }, { "v": 2, "step": 1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", }, ], - {"x": "step", "y": "v", "x_label": "step", "y_label": "v"}, + { + "anchors_y_definitions": [{FILENAME: "f", FIELD: "v"}], + "x": "step", + "y": "v", + "x_label": "step", + "y_label": "v", + }, id="default_x_y", ), pytest.param( @@ -61,23 +63,25 @@ def test_finding_lists(dictionary, expected_result): { "v": 1, "v2": 0.1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "v": 2, "v2": 0.2, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, ], - {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"}, + { + "anchors_y_definitions": [{FILENAME: "f", FIELD: "v2"}], + "x": "v", + "y": "v2", + "x_label": "v", + "y_label": "v2", + }, id="choose_x_y", ), pytest.param( @@ -99,23 +103,25 @@ def test_finding_lists(dictionary, expected_result): { "v": 1, "v2": 0.1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "v": 2, "v2": 0.2, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, ], - {"x": "v", "y": "v2", "x_label": "x", "y_label": "y"}, + { + "anchors_y_definitions": [{FILENAME: "f", FIELD: "v2"}], + "x": "v", + "y": "v2", + "x_label": "x", + "y_label": "y", + }, id="find_in_nested_structure", ), pytest.param( @@ -123,44 +129,36 @@ def test_finding_lists(dictionary, expected_result): {"y": {"f": ["v", "v2"]}}, [ { - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", "dvc_inferred_y_value": 1, "v": 1, "v2": 0.1, "step": 0, }, { - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", "dvc_inferred_y_value": 2, "v": 2, "v2": 0.2, "step": 1, }, { - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", "dvc_inferred_y_value": 0.1, "v2": 0.1, "v": 1, "step": 0, }, { - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", "v": 2, "v2": 0.2, "dvc_inferred_y_value": 0.2, @@ -168,6 +166,10 @@ def test_finding_lists(dictionary, expected_result): }, ], { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v"}, + {FILENAME: "f", FIELD: "v2"}, + ], "x": "step", "y": "dvc_inferred_y_value", "y_label": "y", @@ -189,47 +191,43 @@ def test_finding_lists(dictionary, expected_result): "z": 3, "v": 1, "step": 0, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", }, { "dvc_inferred_y_value": 2, "z": 4, "step": 1, "v": 2, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", }, { "dvc_inferred_y_value": 3, "v": 1, "z": 3, "step": 0, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "z", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "z", }, { "dvc_inferred_y_value": 4, "v": 2, "z": 4, "step": 1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "z", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "z", }, ], { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v"}, + {FILENAME: "f", FIELD: "z"}, + ], "x": "step", "y": "dvc_inferred_y_value", "y_label": "y", @@ -247,32 +245,35 @@ def test_finding_lists(dictionary, expected_result): { "v": 1, "v2": 0.1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "v": 2, "v2": 0.2, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "v": 3, "v2": 0.3, - VERSION_FIELD: { - "revision": "r", - "filename": "f2", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f2", + FIELD: "v2", }, ], - {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"}, + { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v2"}, + {FILENAME: "f2", FIELD: "v2"}, + ], + "x": "v", + "y": "v2", + "x_label": "v", + "y_label": "v2", + }, id="multi_file_json", ), pytest.param( @@ -284,47 +285,43 @@ def test_finding_lists(dictionary, expected_result): "v": 1, "v2": 0.1, "step": 0, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", }, { "dvc_inferred_y_value": 2, "v": 2, "v2": 0.2, "step": 1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v", }, { "dvc_inferred_y_value": 0.1, "v": 1, "v2": 0.1, "step": 0, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "dvc_inferred_y_value": 0.2, "v": 2, "v2": 0.2, "step": 1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, ], { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v"}, + {FILENAME: "f", FIELD: "v2"}, + ], "x": "step", "y": "dvc_inferred_y_value", "x_label": "step", @@ -344,35 +341,34 @@ def test_finding_lists(dictionary, expected_result): "v": 1, "v2": 0.1, "v3": 0.01, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "dvc_inferred_y_value": 0.01, "v": 1, "v2": 0.1, "v3": 0.01, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v3", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v3", }, { "dvc_inferred_y_value": 0.1, "v": 1, "v2": 0.1, - VERSION_FIELD: { - "revision": "r", - "filename": "f2", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f2", + FIELD: "v2", }, ], { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v2"}, + {FILENAME: "f", FIELD: "v3"}, + {FILENAME: "f2", FIELD: "v2"}, + ], "x": "v", "y": "dvc_inferred_y_value", "x_label": "v", @@ -390,23 +386,23 @@ def test_finding_lists(dictionary, expected_result): { "v": 1, "v2": 0.1, - VERSION_FIELD: { - "revision": "r", - "filename": "f", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", }, { "v": 1, "v2": 0.1, - VERSION_FIELD: { - "revision": "r", - "filename": "f2", - "field": "v2", - }, + REVISION: "r", + FILENAME: "f2", + FIELD: "v2", }, ], { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v2"}, + {FILENAME: "f2", FIELD: "v2"}, + ], "x": "v", "y": "v2", "x_label": "v", @@ -414,6 +410,107 @@ def test_finding_lists(dictionary, expected_result): }, id="multi_file_y_same_prefix", ), + pytest.param( + { + "f": {"metric": [{"x1": 1, "v": 0.1}]}, + "f2": {"metric": [{"x2": 100, "v": 0.1}]}, + }, + {"y": {"f": ["v"], "f2": ["v"]}, "x": {"f": "x1", "f2": "x2"}}, + [ + { + "x1": 1, + "v": 0.1, + "dvc_inferred_x_value": 1, + REVISION: "r", + FILENAME: "f", + FIELD: "v", + }, + { + "x2": 100, + "v": 0.1, + "dvc_inferred_x_value": 100, + REVISION: "r", + FILENAME: "f2", + FIELD: "v", + }, + ], + { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v"}, + {FILENAME: "f2", FIELD: "v"}, + ], + "x": "dvc_inferred_x_value", + "y": "v", + "x_label": "x", + "y_label": "v", + }, + id="multiple_x_fields", + ), + pytest.param( + { + "f": { + "metric": [ + {"v": 1, "v2": 0.1, "x1": 100}, + {"v": 2, "v2": 0.2, "x1": 1000}, + ] + }, + "f2": {"metric": [{"x2": -2}, {"x2": -4}]}, + }, + {"y": ["v", "v2"], "x": {"f": "x1", "f2": "x2"}}, + [ + { + "dvc_inferred_x_value": 100, + "dvc_inferred_y_value": 1, + "v": 1, + "v2": 0.1, + "x1": 100, + REVISION: "r", + FILENAME: "f", + FIELD: "v", + }, + { + "dvc_inferred_x_value": 1000, + "dvc_inferred_y_value": 2, + "v": 2, + "v2": 0.2, + "x1": 1000, + REVISION: "r", + FILENAME: "f", + FIELD: "v", + }, + { + "dvc_inferred_x_value": -2, + "dvc_inferred_y_value": 0.1, + "v": 1, + "v2": 0.1, + "x1": 100, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", + }, + { + "dvc_inferred_x_value": -4, + "dvc_inferred_y_value": 0.2, + "v": 2, + "v2": 0.2, + "x1": 1000, + REVISION: "r", + FILENAME: "f", + FIELD: "v2", + }, + ], + { + "anchors_y_definitions": [ + {FILENAME: "f", FIELD: "v"}, + {FILENAME: "f", FIELD: "v2"}, + ], + "x": "dvc_inferred_x_value", + "y": "dvc_inferred_y_value", + "x_label": "x", + "y_label": "y", + }, + id="y_list_x_dict", + ), ], ) def test_convert(