From 13f8c23644c73e4096d94662f82f00777cb6d1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Mon, 6 Sep 2021 11:11:07 +0200 Subject: [PATCH 1/8] plots: vega: make interactive linear default plot --- dvc/repo/plots/template.py | 207 ++++++++++++++++----------------- tests/unit/render/test_vega.py | 57 +++++---- 2 files changed, 140 insertions(+), 124 deletions(-) diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index 187e5b7f03..9c147924e5 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -108,28 +108,101 @@ def _check_field_exists(data, field): class DefaultTemplate(Template): DEFAULT_NAME = "default" - DEFAULT_CONTENT = { "$schema": "https://vega.github.io/schema/vega-lite/v4.json", "data": {"values": Template.anchor("data")}, "title": Template.anchor("title"), "width": 300, "height": 300, - "mark": {"type": "line"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), + "layer": [ + { + "encoding": { + "x": { + "field": Template.anchor("x"), + "type": "quantitative", + "title": Template.anchor("x_label"), + }, + "y": { + "field": Template.anchor("y"), + "type": "quantitative", + "title": Template.anchor("y_label"), + "scale": {"zero": False}, + }, + "color": {"field": "rev", "type": "nominal"}, + }, + "layer": [ + {"mark": "line"}, + { + "selection": { + "label": { + "type": "single", + "nearest": True, + "on": "mouseover", + "encodings": ["x"], + "empty": "none", + "clear": "mouseout", + } + }, + "mark": "point", + "encoding": { + "opacity": { + "condition": { + "selection": "label", + "value": 1, + }, + "value": 0, + } + }, + }, + ], }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, + { + "transform": [{"filter": {"selection": "label"}}], + "layer": [ + { + "mark": {"type": "rule", "color": "gray"}, + "encoding": { + "x": { + "field": Template.anchor("x"), + "type": "quantitative", + } + }, + }, + { + "encoding": { + "text": { + "type": "quantitative", + "field": Template.anchor("y"), + }, + "x": { + "field": Template.anchor("x"), + "type": "quantitative", + }, + "y": { + "field": Template.anchor("y"), + "type": "quantitative", + }, + }, + "layer": [ + { + "mark": { + "type": "text", + "align": "left", + "dx": 5, + "dy": -5, + }, + "encoding": { + "color": { + "type": "nominal", + "field": "rev", + } + }, + } + ], + }, + ], }, - "color": {"field": "rev", "type": "nominal"}, - }, + ], } @@ -426,8 +499,8 @@ class SmoothLinearTemplate(Template): } -class LinearTemplate(Template): - DEFAULT_NAME = "linear" +class SimpleLinearTemplate(Template): + DEFAULT_NAME = "simple" DEFAULT_CONTENT = { "$schema": "https://vega.github.io/schema/vega-lite/v4.json", @@ -435,95 +508,21 @@ class LinearTemplate(Template): "title": Template.anchor("title"), "width": 300, "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "layer": [ - {"mark": "line"}, - { - "selection": { - "label": { - "type": "single", - "nearest": True, - "on": "mouseover", - "encodings": ["x"], - "empty": "none", - "clear": "mouseout", - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1, - }, - "value": 0, - } - }, - }, - ], + "mark": {"type": "line"}, + "encoding": { + "x": { + "field": Template.anchor("x"), + "type": "quantitative", + "title": Template.anchor("x_label"), }, - { - "transform": [{"filter": {"selection": "label"}}], - "layer": [ - { - "mark": {"type": "rule", "color": "gray"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - } - }, - }, - { - "encoding": { - "text": { - "type": "quantitative", - "field": Template.anchor("y"), - }, - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - }, - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5, - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev", - } - }, - } - ], - }, - ], + "y": { + "field": Template.anchor("y"), + "type": "quantitative", + "title": Template.anchor("y_label"), + "scale": {"zero": False}, }, - ], + "color": {"field": "rev", "type": "nominal"}, + }, } @@ -531,7 +530,7 @@ class PlotTemplates: TEMPLATES_DIR = "plots" TEMPLATES = [ DefaultTemplate, - LinearTemplate, + SimpleLinearTemplate, ConfusionTemplate, NormalizedConfusionTemplate, ScatterTemplate, diff --git a/tests/unit/render/test_vega.py b/tests/unit/render/test_vega.py index 41002e38b6..dec89bd50e 100644 --- a/tests/unit/render/test_vega.py +++ b/tests/unit/render/test_vega.py @@ -3,6 +3,7 @@ from collections import OrderedDict import pytest +from funcy import first from dvc.render.utils import find_vega, group_by_filename from dvc.render.vega import ( @@ -117,10 +118,12 @@ def test_one_column(tmp_dir, scm, dvc): {"val": 2, INDEX_FIELD: 0, REVISION_FIELD: "workspace"}, {"val": 3, INDEX_FIELD: 1, REVISION_FIELD: "workspace"}, ] - assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD - assert plot_content["encoding"]["y"]["field"] == "val" - assert plot_content["encoding"]["x"]["title"] == "x_title" - assert plot_content["encoding"]["y"]["title"] == "y_title" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD + ) + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "val" + assert first(plot_content["layer"])["encoding"]["x"]["title"] == "x_title" + assert first(plot_content["layer"])["encoding"]["y"]["title"] == "y_title" def test_multiple_columns(tmp_dir, scm, dvc): @@ -152,8 +155,10 @@ def test_multiple_columns(tmp_dir, scm, dvc): "second_val": 300, }, ] - assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD - assert plot_content["encoding"]["y"]["field"] == "val" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD + ) + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "val" def test_choose_axes(tmp_dir, scm, dvc): @@ -184,8 +189,12 @@ def test_choose_axes(tmp_dir, scm, dvc): "second_val": 300, }, ] - assert plot_content["encoding"]["x"]["field"] == "first_val" - assert plot_content["encoding"]["y"]["field"] == "second_val" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == "first_val" + ) + assert ( + first(plot_content["layer"])["encoding"]["y"]["field"] == "second_val" + ) def test_confusion(tmp_dir, dvc): @@ -250,8 +259,10 @@ def test_multiple_revs_default(tmp_dir, scm, dvc): {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"}, {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"}, ] - assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD - assert plot_content["encoding"]["y"]["field"] == "y" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD + ) + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y" def test_metric_missing(tmp_dir, scm, dvc, caplog): @@ -270,8 +281,10 @@ def test_metric_missing(tmp_dir, scm, dvc, caplog): {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v2"}, {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v2"}, ] - assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD - assert plot_content["encoding"]["y"]["field"] == "y" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD + ) + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y" def test_custom_template(tmp_dir, scm, dvc, custom_template): @@ -288,8 +301,8 @@ def test_custom_template(tmp_dir, scm, dvc, custom_template): {"a": 1, "b": 2, REVISION_FIELD: "workspace"}, {"a": 2, "b": 3, REVISION_FIELD: "workspace"}, ] - assert plot_content["encoding"]["x"]["field"] == "a" - assert plot_content["encoding"]["y"]["field"] == "b" + assert first(plot_content["layer"])["encoding"]["x"]["field"] == "a" + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "b" def test_raise_on_no_template(tmp_dir, dvc): @@ -334,8 +347,8 @@ def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template): {"b": 2, "c": 3, REVISION_FIELD: "workspace"}, {"b": 3, "c": 4, REVISION_FIELD: "workspace"}, ] - assert plot_content["encoding"]["x"]["field"] == "b" - assert plot_content["encoding"]["y"]["field"] == "c" + assert first(plot_content["layer"])["encoding"]["x"]["field"] == "b" + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "c" def test_plot_default_choose_column(tmp_dir, scm, dvc): @@ -353,8 +366,10 @@ def test_plot_default_choose_column(tmp_dir, scm, dvc): {INDEX_FIELD: 0, "b": 2, REVISION_FIELD: "workspace"}, {INDEX_FIELD: 1, "b": 3, REVISION_FIELD: "workspace"}, ] - assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD - assert plot_content["encoding"]["y"]["field"] == "b" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD + ) + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "b" def test_raise_on_wrong_field(tmp_dir, scm, dvc): @@ -432,5 +447,7 @@ def test_find_vega(tmp_dir, dvc): {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"}, {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"}, ] - assert plot_content["encoding"]["x"]["field"] == INDEX_FIELD - assert plot_content["encoding"]["y"]["field"] == "y" + assert ( + first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD + ) + assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y" From a6c108a95bfa7e26140c15b73cc148b8d9cc9981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Tue, 7 Sep 2021 16:55:21 +0200 Subject: [PATCH 2/8] plots: stop dumping default templates --- dvc/repo/init.py | 2 -- dvc/repo/plots/template.py | 26 -------------------------- tests/conftest.py | 4 ++-- tests/unit/render/test_vega.py | 8 ++++---- 4 files changed, 6 insertions(+), 34 deletions(-) diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 4243066bb7..97c729f3f0 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -76,8 +76,6 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): proj = Repo(root_dir) - proj.plots.templates.init() - scm.add( [config.files["repo"], dvcignore, proj.plots.templates.templates_dir] ) diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index 9c147924e5..a799f0b886 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -545,37 +545,11 @@ def get_template(self, path): if os.path.exists(path): return path - if self.dvc_dir and os.path.exists(self.dvc_dir): - t_path = os.path.join(self.templates_dir, path) - if os.path.exists(t_path): - return t_path - - all_templates = [ - os.path.join(root, file) - for root, _, files in os.walk(self.templates_dir) - for file in files - ] - matches = [ - template - for template in all_templates - if os.path.splitext(template)[0] == t_path - ] - if matches: - assert len(matches) == 1 - return matches[0] - raise TemplateNotFoundError(path) def __init__(self, dvc_dir): self.dvc_dir = dvc_dir - def init(self): - from dvc.utils.fs import makedirs - - makedirs(self.templates_dir, exist_ok=True) - for t in self.TEMPLATES: - self._dump(t()) - def _dump(self, template): path = os.path.join(self.templates_dir, template.filename) with open(path, "w") as fd: diff --git a/tests/conftest.py b/tests/conftest.py index 69aeab8e8d..edbc83053b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -161,10 +161,10 @@ def pytest_configure(config): @pytest.fixture() def custom_template(tmp_dir, dvc): - import shutil + from dvc.repo.plots.template import SimpleLinearTemplate template = tmp_dir / "custom_template.json" - shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", template) + template.write_text(SimpleLinearTemplate().content) return template diff --git a/tests/unit/render/test_vega.py b/tests/unit/render/test_vega.py index dec89bd50e..0b3d936c67 100644 --- a/tests/unit/render/test_vega.py +++ b/tests/unit/render/test_vega.py @@ -301,8 +301,8 @@ def test_custom_template(tmp_dir, scm, dvc, custom_template): {"a": 1, "b": 2, REVISION_FIELD: "workspace"}, {"a": 2, "b": 3, REVISION_FIELD: "workspace"}, ] - assert first(plot_content["layer"])["encoding"]["x"]["field"] == "a" - assert first(plot_content["layer"])["encoding"]["y"]["field"] == "b" + assert plot_content["encoding"]["x"]["field"] == "a" + assert plot_content["encoding"]["y"]["field"] == "b" def test_raise_on_no_template(tmp_dir, dvc): @@ -347,8 +347,8 @@ def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template): {"b": 2, "c": 3, REVISION_FIELD: "workspace"}, {"b": 3, "c": 4, REVISION_FIELD: "workspace"}, ] - assert first(plot_content["layer"])["encoding"]["x"]["field"] == "b" - assert first(plot_content["layer"])["encoding"]["y"]["field"] == "c" + assert plot_content["encoding"]["x"]["field"] == "b" + assert plot_content["encoding"]["y"]["field"] == "c" def test_plot_default_choose_column(tmp_dir, scm, dvc): From def466449408ad8e154f81a281ac2cba180ec654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Thu, 16 Sep 2021 14:21:30 +0200 Subject: [PATCH 3/8] plots: move templates to package initial --- MANIFEST.in | 2 + dvc/render/vega.py | 2 +- dvc/repo/init.py | 2 + dvc/repo/plots/__init__.py | 2 +- dvc/repo/plots/template.py | 542 +++--------------- dvc/repo/plots/templates/confusion.json | 107 ++++ .../plots/templates/confusion_normalized.json | 112 ++++ dvc/repo/plots/templates/linear.json | 116 ++++ dvc/repo/plots/templates/scatter.json | 104 ++++ dvc/repo/plots/templates/simple.json | 31 + dvc/repo/plots/templates/smooth.json | 39 ++ tests/conftest.py | 8 +- 12 files changed, 598 insertions(+), 469 deletions(-) create mode 100644 dvc/repo/plots/templates/confusion.json create mode 100644 dvc/repo/plots/templates/confusion_normalized.json create mode 100644 dvc/repo/plots/templates/linear.json create mode 100644 dvc/repo/plots/templates/scatter.json create mode 100644 dvc/repo/plots/templates/simple.json create mode 100644 dvc/repo/plots/templates/smooth.json diff --git a/MANIFEST.in b/MANIFEST.in index 03d2f17a9a..0072fe34bf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ include fastentrypoints.py include LICENSE +recursive-include requirements *.txt +recursive-include dvc/repo/plots/templates *.json diff --git a/dvc/render/vega.py b/dvc/render/vega.py index 7e346f9f14..2813988267 100644 --- a/dvc/render/vega.py +++ b/dvc/render/vega.py @@ -147,7 +147,7 @@ def _datapoints(self, props: Dict): def get_vega(self) -> Optional[str]: props = self._squash_props() - template = self.templates.load(props.get("template") or "default") + template = self.templates.load(props.get("template", None)) if not props.get("x") and template.has_anchor("x"): props["append_index"] = True diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 97c729f3f0..4243066bb7 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -76,6 +76,8 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): proj = Repo(root_dir) + proj.plots.templates.init() + scm.add( [config.files["repo"], dvcignore, proj.plots.templates.templates_dir] ) diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index cdb4dcac55..e58fc0f79b 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -161,7 +161,7 @@ def modify(self, path, props=None, unset=None): props = props or {} template = props.get("template") if template: - self.templates.get_template(template) + self.templates.load(template) (out,) = self.repo.find_outs_by_path(path) if not out.plot and unset is not None: diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index a799f0b886..d3b00e9e19 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -1,15 +1,18 @@ import json import os -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional from funcy import cached_property from dvc.exceptions import DvcException +if TYPE_CHECKING: + from dvc.types import StrPath + class TemplateNotFoundError(DvcException): - def __init__(self, path): - super().__init__(f"Template '{path}' not found.") + def __init__(self, name): + super().__init__(f"Template '{name}' not found.") class BadTemplateError(DvcException): @@ -30,24 +33,10 @@ class Template: ANCHOR = "" DEFAULT_CONTENT: Optional[Dict[str, Any]] = None - DEFAULT_NAME: Optional[str] = None - - def __init__(self, content=None, name=None): - if content: - self.content = content - else: - self.content = ( - json.dumps( - self.DEFAULT_CONTENT, - indent=self.INDENT, - separators=self.SEPARATORS, - ) - + "\n" - ) - self.name = name or self.DEFAULT_NAME - assert self.content and self.name - self.filename = self.name + self.EXTENSION + def __init__(self, content, name): + self.content = content + self.name = name def render(self, data, props=None): props = props or {} @@ -55,7 +44,7 @@ def render(self, data, props=None): if self._anchor_str("data") not in self.content: anchor = self.anchor("data") raise BadTemplateError( - f"Template '{self.filename}' is not using '{anchor}' anchor" + f"Template '{self.name}' is not using '{anchor}' anchor" ) if props.get("x"): @@ -106,465 +95,88 @@ def _check_field_exists(data, field): raise NoFieldInDataError(field) -class DefaultTemplate(Template): - DEFAULT_NAME = "default" - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "width": 300, - "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "layer": [ - {"mark": "line"}, - { - "selection": { - "label": { - "type": "single", - "nearest": True, - "on": "mouseover", - "encodings": ["x"], - "empty": "none", - "clear": "mouseout", - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1, - }, - "value": 0, - } - }, - }, - ], - }, - { - "transform": [{"filter": {"selection": "label"}}], - "layer": [ - { - "mark": {"type": "rule", "color": "gray"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - } - }, - }, - { - "encoding": { - "text": { - "type": "quantitative", - "field": Template.anchor("y"), - }, - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - }, - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5, - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev", - } - }, - } - ], - }, - ], - }, - ], - } - - -class ConfusionTemplate(Template): - DEFAULT_NAME = "confusion" - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "facet": {"field": "rev", "type": "nominal"}, - "spec": { - "transform": [ - { - "aggregate": [{"op": "count", "as": "xy_count"}], - "groupby": [Template.anchor("y"), Template.anchor("x")], - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("y")], - "key": Template.anchor("x"), - "value": 0, - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("x")], - "key": Template.anchor("y"), - "value": 0, - }, - { - "joinaggregate": [ - {"op": "max", "field": "xy_count", "as": "max_count"} - ], - "groupby": [], - }, - { - "calculate": "datum.xy_count / datum.max_count", - "as": "percent_of_max", - }, - ], - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("y_label"), - }, - }, - "layer": [ - { - "mark": "rect", - "width": 300, - "height": 300, - "encoding": { - "color": { - "field": "xy_count", - "type": "quantitative", - "title": "", - "scale": {"domainMin": 0, "nice": True}, - } - }, - }, - { - "mark": "text", - "encoding": { - "text": {"field": "xy_count", "type": "quantitative"}, - "color": { - "condition": { - "test": "datum.percent_of_max > 0.5", - "value": "white", - }, - "value": "black", - }, - }, - }, - ], - }, - } - - -class NormalizedConfusionTemplate(Template): - DEFAULT_NAME = "confusion_normalized" - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "facet": {"field": "rev", "type": "nominal"}, - "spec": { - "transform": [ - { - "aggregate": [{"op": "count", "as": "xy_count"}], - "groupby": [Template.anchor("y"), Template.anchor("x")], - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("y")], - "key": Template.anchor("x"), - "value": 0, - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("x")], - "key": Template.anchor("y"), - "value": 0, - }, - { - "joinaggregate": [ - {"op": "sum", "field": "xy_count", "as": "sum_y"} - ], - "groupby": [Template.anchor("y")], - }, - { - "calculate": "datum.xy_count / datum.sum_y", - "as": "percent_of_y", - }, - ], - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("y_label"), - }, - }, - "layer": [ - { - "mark": "rect", - "width": 300, - "height": 300, - "encoding": { - "color": { - "field": "percent_of_y", - "type": "quantitative", - "title": "", - "scale": {"domain": [0, 1]}, - } - }, - }, - { - "mark": "text", - "encoding": { - "text": { - "field": "percent_of_y", - "type": "quantitative", - "format": ".2f", - }, - "color": { - "condition": { - "test": "datum.percent_of_y > 0.5", - "value": "white", - }, - "value": "black", - }, - }, - }, - ], - }, - } - - -class ScatterTemplate(Template): - DEFAULT_NAME = "scatter" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "width": 300, - "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "layer": [ - {"mark": "point"}, - { - "selection": { - "label": { - "type": "single", - "nearest": True, - "on": "mouseover", - "encodings": ["x"], - "empty": "none", - "clear": "mouseout", - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1, - }, - "value": 0, - } - }, - }, - ], - }, - { - "transform": [{"filter": {"selection": "label"}}], - "layer": [ - { - "encoding": { - "text": { - "type": "quantitative", - "field": Template.anchor("y"), - }, - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - }, - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5, - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev", - } - }, - } - ], - } - ], - }, - ], - } - - -class SmoothLinearTemplate(Template): - DEFAULT_NAME = "smooth" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "mark": {"type": "line"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "transform": [ - { - "loess": Template.anchor("y"), - "on": Template.anchor("x"), - "groupby": ["rev"], - "bandwidth": 0.3, - } - ], - } - - -class SimpleLinearTemplate(Template): - DEFAULT_NAME = "simple" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "width": 300, - "height": 300, - "mark": {"type": "line"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - } +# TODO upgrade templates schema class PlotTemplates: TEMPLATES_DIR = "plots" - TEMPLATES = [ - DefaultTemplate, - SimpleLinearTemplate, - ConfusionTemplate, - NormalizedConfusionTemplate, - ScatterTemplate, - SmoothLinearTemplate, - ] + PKG_TEMPLATES_DIR = "templates" @cached_property def templates_dir(self): return os.path.join(self.dvc_dir, self.TEMPLATES_DIR) - def get_template(self, path): - if os.path.exists(path): - return path + @staticmethod + def _find(templates, template_name): + for template in templates: + if ( + template_name == template + or template_name + ".json" == template + ): + return template + return None + + def _find_in_project(self, name: str) -> Optional["StrPath"]: + if os.path.exists(name): + return name + + if os.path.exists(self.templates_dir): + templates = [ + os.path.join(root, file) + for root, _, files in os.walk(self.templates_dir) + for file in files + ] + found = self._find(templates, name) + if found: + return os.path.join(self.templates_dir, found) + return None - raise TemplateNotFoundError(path) + @staticmethod + def _load_from_pkg(name): + import pkg_resources # type: ignore + + if pkg_resources.resource_exists( + __name__, PlotTemplates.PKG_TEMPLATES_DIR + ): + templates = pkg_resources.resource_listdir( + __name__, PlotTemplates.PKG_TEMPLATES_DIR + ) + found = PlotTemplates._find(templates, name) + if found: + return pkg_resources.resource_string( + __name__, f"{PlotTemplates.PKG_TEMPLATES_DIR}/{found}" + ).decode("utf-8") + return None + + def load(self, name: str = None) -> Template: + + if name is not None: + template_path = self._find_in_project(name) + if template_path: + with open(template_path, "r") as fd: + content = fd.read() + return Template(content, name) + else: + name = "linear" + + content = self._load_from_pkg(name) + if content: + return Template(content, name) + + raise TemplateNotFoundError(name) def __init__(self, dvc_dir): self.dvc_dir = dvc_dir + def init(self): + from dvc.utils.fs import makedirs + + makedirs(self.templates_dir, exist_ok=True) + # TODO copy to `.dvc/plots` + # for t in self.TEMPLATES: + # self._dump(t()) + def _dump(self, template): path = os.path.join(self.templates_dir, template.filename) with open(path, "w") as fd: fd.write(template.content) - - def load(self, name): - try: - path = self.get_template(name) - - with open(path) as fd: - content = fd.read() - - return Template(content, name=name) - except TemplateNotFoundError: - for template in self.TEMPLATES: - if template.DEFAULT_NAME == name: - return template() - raise diff --git a/dvc/repo/plots/templates/confusion.json b/dvc/repo/plots/templates/confusion.json new file mode 100644 index 0000000000..af1b48d031 --- /dev/null +++ b/dvc/repo/plots/templates/confusion.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": { + "values": "" + }, + "title": "", + "facet": { + "field": "rev", + "type": "nominal" + }, + "spec": { + "transform": [ + { + "aggregate": [ + { + "op": "count", + "as": "xy_count" + } + ], + "groupby": [ + "", + "" + ] + }, + { + "impute": "xy_count", + "groupby": [ + "rev", + "" + ], + "key": "", + "value": 0 + }, + { + "impute": "xy_count", + "groupby": [ + "rev", + "" + ], + "key": "", + "value": 0 + }, + { + "joinaggregate": [ + { + "op": "max", + "field": "xy_count", + "as": "max_count" + } + ], + "groupby": [] + }, + { + "calculate": "datum.xy_count / datum.max_count", + "as": "percent_of_max" + } + ], + "encoding": { + "x": { + "field": "", + "type": "nominal", + "sort": "ascending", + "title": "" + }, + "y": { + "field": "", + "type": "nominal", + "sort": "ascending", + "title": "" + } + }, + "layer": [ + { + "mark": "rect", + "width": 300, + "height": 300, + "encoding": { + "color": { + "field": "xy_count", + "type": "quantitative", + "title": "", + "scale": { + "domainMin": 0, + "nice": true + } + } + } + }, + { + "mark": "text", + "encoding": { + "text": { + "field": "xy_count", + "type": "quantitative" + }, + "color": { + "condition": { + "test": "datum.percent_of_max > 0.5", + "value": "white" + }, + "value": "black" + } + } + } + ] + } +} diff --git a/dvc/repo/plots/templates/confusion_normalized.json b/dvc/repo/plots/templates/confusion_normalized.json new file mode 100644 index 0000000000..1d38849f48 --- /dev/null +++ b/dvc/repo/plots/templates/confusion_normalized.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": { + "values": "" + }, + "title": "", + "facet": { + "field": "rev", + "type": "nominal" + }, + "spec": { + "transform": [ + { + "aggregate": [ + { + "op": "count", + "as": "xy_count" + } + ], + "groupby": [ + "", + "" + ] + }, + { + "impute": "xy_count", + "groupby": [ + "rev", + "" + ], + "key": "", + "value": 0 + }, + { + "impute": "xy_count", + "groupby": [ + "rev", + "" + ], + "key": "", + "value": 0 + }, + { + "joinaggregate": [ + { + "op": "sum", + "field": "xy_count", + "as": "sum_y" + } + ], + "groupby": [ + "" + ] + }, + { + "calculate": "datum.xy_count / datum.sum_y", + "as": "percent_of_y" + } + ], + "encoding": { + "x": { + "field": "", + "type": "nominal", + "sort": "ascending", + "title": "" + }, + "y": { + "field": "", + "type": "nominal", + "sort": "ascending", + "title": "" + } + }, + "layer": [ + { + "mark": "rect", + "width": 300, + "height": 300, + "encoding": { + "color": { + "field": "percent_of_y", + "type": "quantitative", + "title": "", + "scale": { + "domain": [ + 0, + 1 + ] + } + } + } + }, + { + "mark": "text", + "encoding": { + "text": { + "field": "percent_of_y", + "type": "quantitative", + "format": ".2f" + }, + "color": { + "condition": { + "test": "datum.percent_of_y > 0.5", + "value": "white" + }, + "value": "black" + } + } + } + ] + } +} diff --git a/dvc/repo/plots/templates/linear.json b/dvc/repo/plots/templates/linear.json new file mode 100644 index 0000000000..65549f9e01 --- /dev/null +++ b/dvc/repo/plots/templates/linear.json @@ -0,0 +1,116 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": { + "values": "" + }, + "title": "", + "width": 300, + "height": 300, + "layer": [ + { + "encoding": { + "x": { + "field": "", + "type": "quantitative", + "title": "" + }, + "y": { + "field": "", + "type": "quantitative", + "title": "", + "scale": { + "zero": false + } + }, + "color": { + "field": "rev", + "type": "nominal" + } + }, + "layer": [ + { + "mark": "line" + }, + { + "selection": { + "label": { + "type": "single", + "nearest": true, + "on": "mouseover", + "encodings": [ + "x" + ], + "empty": "none", + "clear": "mouseout" + } + }, + "mark": "point", + "encoding": { + "opacity": { + "condition": { + "selection": "label", + "value": 1 + }, + "value": 0 + } + } + } + ] + }, + { + "transform": [ + { + "filter": { + "selection": "label" + } + } + ], + "layer": [ + { + "mark": { + "type": "rule", + "color": "gray" + }, + "encoding": { + "x": { + "field": "", + "type": "quantitative" + } + } + }, + { + "encoding": { + "text": { + "type": "quantitative", + "field": "" + }, + "x": { + "field": "", + "type": "quantitative" + }, + "y": { + "field": "", + "type": "quantitative" + } + }, + "layer": [ + { + "mark": { + "type": "text", + "align": "left", + "dx": 5, + "dy": -5 + }, + "encoding": { + "color": { + "type": "nominal", + "field": "rev" + } + } + } + ] + } + ] + } + ] +} diff --git a/dvc/repo/plots/templates/scatter.json b/dvc/repo/plots/templates/scatter.json new file mode 100644 index 0000000000..9af9304c64 --- /dev/null +++ b/dvc/repo/plots/templates/scatter.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": { + "values": "" + }, + "title": "", + "width": 300, + "height": 300, + "layer": [ + { + "encoding": { + "x": { + "field": "", + "type": "quantitative", + "title": "" + }, + "y": { + "field": "", + "type": "quantitative", + "title": "", + "scale": { + "zero": false + } + }, + "color": { + "field": "rev", + "type": "nominal" + } + }, + "layer": [ + { + "mark": "point" + }, + { + "selection": { + "label": { + "type": "single", + "nearest": true, + "on": "mouseover", + "encodings": [ + "x" + ], + "empty": "none", + "clear": "mouseout" + } + }, + "mark": "point", + "encoding": { + "opacity": { + "condition": { + "selection": "label", + "value": 1 + }, + "value": 0 + } + } + } + ] + }, + { + "transform": [ + { + "filter": { + "selection": "label" + } + } + ], + "layer": [ + { + "encoding": { + "text": { + "type": "quantitative", + "field": "" + }, + "x": { + "field": "", + "type": "quantitative" + }, + "y": { + "field": "", + "type": "quantitative" + } + }, + "layer": [ + { + "mark": { + "type": "text", + "align": "left", + "dx": 5, + "dy": -5 + }, + "encoding": { + "color": { + "type": "nominal", + "field": "rev" + } + } + } + ] + } + ] + } + ] +} diff --git a/dvc/repo/plots/templates/simple.json b/dvc/repo/plots/templates/simple.json new file mode 100644 index 0000000000..9cf71ce0a2 --- /dev/null +++ b/dvc/repo/plots/templates/simple.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": { + "values": "" + }, + "title": "", + "width": 300, + "height": 300, + "mark": { + "type": "line" + }, + "encoding": { + "x": { + "field": "", + "type": "quantitative", + "title": "" + }, + "y": { + "field": "", + "type": "quantitative", + "title": "", + "scale": { + "zero": false + } + }, + "color": { + "field": "rev", + "type": "nominal" + } + } +} diff --git a/dvc/repo/plots/templates/smooth.json b/dvc/repo/plots/templates/smooth.json new file mode 100644 index 0000000000..d497ce75e9 --- /dev/null +++ b/dvc/repo/plots/templates/smooth.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": { + "values": "" + }, + "title": "", + "mark": { + "type": "line" + }, + "encoding": { + "x": { + "field": "", + "type": "quantitative", + "title": "" + }, + "y": { + "field": "", + "type": "quantitative", + "title": "", + "scale": { + "zero": false + } + }, + "color": { + "field": "rev", + "type": "nominal" + } + }, + "transform": [ + { + "loess": "", + "on": "", + "groupby": [ + "rev" + ], + "bandwidth": 0.3 + } + ] +} diff --git a/tests/conftest.py b/tests/conftest.py index edbc83053b..d859191bc3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -161,10 +161,14 @@ def pytest_configure(config): @pytest.fixture() def custom_template(tmp_dir, dvc): - from dvc.repo.plots.template import SimpleLinearTemplate + import pkg_resources + + content = pkg_resources.resource_string( + "dvc.repo.plots", "templates/simple.json" + ).decode("utf-8") template = tmp_dir / "custom_template.json" - template.write_text(SimpleLinearTemplate().content) + template.write_text(content) return template From b9d5897b089c552e7e50b88113f3a42e328cd481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Mon, 20 Sep 2021 13:48:56 +0200 Subject: [PATCH 4/8] plots: update default plots schema --- dvc/repo/plots/template.py | 10 +--------- dvc/repo/plots/templates/confusion.json | 2 +- dvc/repo/plots/templates/confusion_normalized.json | 2 +- dvc/repo/plots/templates/linear.json | 2 +- dvc/repo/plots/templates/scatter.json | 2 +- dvc/repo/plots/templates/simple.json | 2 +- dvc/repo/plots/templates/smooth.json | 2 +- 7 files changed, 7 insertions(+), 15 deletions(-) diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index d3b00e9e19..d121be4128 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -1,6 +1,6 @@ import json import os -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Optional from funcy import cached_property @@ -32,8 +32,6 @@ class Template: EXTENSION = ".json" ANCHOR = "" - DEFAULT_CONTENT: Optional[Dict[str, Any]] = None - def __init__(self, content, name): self.content = content self.name = name @@ -95,9 +93,6 @@ def _check_field_exists(data, field): raise NoFieldInDataError(field) -# TODO upgrade templates schema - - class PlotTemplates: TEMPLATES_DIR = "plots" PKG_TEMPLATES_DIR = "templates" @@ -172,9 +167,6 @@ def init(self): from dvc.utils.fs import makedirs makedirs(self.templates_dir, exist_ok=True) - # TODO copy to `.dvc/plots` - # for t in self.TEMPLATES: - # self._dump(t()) def _dump(self, template): path = os.path.join(self.templates_dir, template.filename) diff --git a/dvc/repo/plots/templates/confusion.json b/dvc/repo/plots/templates/confusion.json index af1b48d031..84ec022f81 100644 --- a/dvc/repo/plots/templates/confusion.json +++ b/dvc/repo/plots/templates/confusion.json @@ -1,5 +1,5 @@ { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "data": { "values": "" }, diff --git a/dvc/repo/plots/templates/confusion_normalized.json b/dvc/repo/plots/templates/confusion_normalized.json index 1d38849f48..92c77739c4 100644 --- a/dvc/repo/plots/templates/confusion_normalized.json +++ b/dvc/repo/plots/templates/confusion_normalized.json @@ -1,5 +1,5 @@ { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "data": { "values": "" }, diff --git a/dvc/repo/plots/templates/linear.json b/dvc/repo/plots/templates/linear.json index 65549f9e01..970dc929ad 100644 --- a/dvc/repo/plots/templates/linear.json +++ b/dvc/repo/plots/templates/linear.json @@ -1,5 +1,5 @@ { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "data": { "values": "" }, diff --git a/dvc/repo/plots/templates/scatter.json b/dvc/repo/plots/templates/scatter.json index 9af9304c64..6e8cf5b48e 100644 --- a/dvc/repo/plots/templates/scatter.json +++ b/dvc/repo/plots/templates/scatter.json @@ -1,5 +1,5 @@ { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "data": { "values": "" }, diff --git a/dvc/repo/plots/templates/simple.json b/dvc/repo/plots/templates/simple.json index 9cf71ce0a2..1cebce9ba9 100644 --- a/dvc/repo/plots/templates/simple.json +++ b/dvc/repo/plots/templates/simple.json @@ -1,5 +1,5 @@ { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "data": { "values": "" }, diff --git a/dvc/repo/plots/templates/smooth.json b/dvc/repo/plots/templates/smooth.json index d497ce75e9..42b1ecfff2 100644 --- a/dvc/repo/plots/templates/smooth.json +++ b/dvc/repo/plots/templates/smooth.json @@ -1,5 +1,5 @@ { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "data": { "values": "" }, From 5809a894513bec03862880b14f8a5c1f657ab5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Mon, 20 Sep 2021 14:36:16 +0200 Subject: [PATCH 5/8] plots: write defaults on init --- dvc/render/utils.py | 2 +- dvc/repo/plots/template.py | 35 +++++++++++++++++++------------- tests/unit/render/test_render.py | 8 ++++++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/dvc/render/utils.py b/dvc/render/utils.py index e9b93021b8..ff74d210f4 100644 --- a/dvc/render/utils.py +++ b/dvc/render/utils.py @@ -1,7 +1,7 @@ import os.path from typing import Dict, List -import dpath +import dpath.util def get_files(data: Dict) -> List: diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index d121be4128..dc747b0ca4 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -1,7 +1,8 @@ import json import os -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Iterable, Optional +import pkg_resources # type: ignore from funcy import cached_property from dvc.exceptions import DvcException @@ -127,20 +128,23 @@ def _find_in_project(self, name: str) -> Optional["StrPath"]: return None @staticmethod - def _load_from_pkg(name): - import pkg_resources # type: ignore - + def _get_templates() -> Iterable[str]: if pkg_resources.resource_exists( __name__, PlotTemplates.PKG_TEMPLATES_DIR ): - templates = pkg_resources.resource_listdir( + return pkg_resources.resource_listdir( __name__, PlotTemplates.PKG_TEMPLATES_DIR ) - found = PlotTemplates._find(templates, name) - if found: - return pkg_resources.resource_string( - __name__, f"{PlotTemplates.PKG_TEMPLATES_DIR}/{found}" - ).decode("utf-8") + return [] + + @staticmethod + def _load_from_pkg(name): + templates = PlotTemplates._get_templates() + found = PlotTemplates._find(templates, name) + if found: + return pkg_resources.resource_string( + __name__, f"{PlotTemplates.PKG_TEMPLATES_DIR}/{found}" + ).decode("utf-8") return None def load(self, name: str = None) -> Template: @@ -168,7 +172,10 @@ def init(self): makedirs(self.templates_dir, exist_ok=True) - def _dump(self, template): - path = os.path.join(self.templates_dir, template.filename) - with open(path, "w") as fd: - fd.write(template.content) + templates = self._get_templates() + for template in templates: + content = pkg_resources.resource_string( + __name__, f"{PlotTemplates.PKG_TEMPLATES_DIR}/{template}" + ) + with open(os.path.join(self.templates_dir, template), "wb") as fd: + fd.write(content) diff --git a/tests/unit/render/test_render.py b/tests/unit/render/test_render.py index 55496b6f3e..3703bf54ab 100644 --- a/tests/unit/render/test_render.py +++ b/tests/unit/render/test_render.py @@ -63,5 +63,9 @@ def test_render(tmp_dir, dvc): index_content = index_path.read_text() file_vega = find_vega(dvc, data, "file.json") some_vega = find_vega(dvc, data, "some.csv") - assert file_vega in index_content.strip() - assert some_vega in index_content.strip() + + def clean(txt: str) -> str: + return txt.replace("\n", "").replace("\r", "").replace(" ", "") + + assert clean(file_vega) in clean(index_content) + assert clean(some_vega) in clean(index_content) From 246ca0418521f663504ea51c1108094dd8d0dbcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Fri, 8 Oct 2021 14:39:25 +0200 Subject: [PATCH 6/8] templates: migrate to importlib.resources --- dvc/repo/plots/template.py | 36 +++++++++++++++++++++++++----------- tests/conftest.py | 10 ++++++---- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index dc747b0ca4..d604d25ef5 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -2,7 +2,7 @@ import os from typing import TYPE_CHECKING, Iterable, Optional -import pkg_resources # type: ignore +import importlib_resources from funcy import cached_property from dvc.exceptions import DvcException @@ -129,12 +129,17 @@ def _find_in_project(self, name: str) -> Optional["StrPath"]: @staticmethod def _get_templates() -> Iterable[str]: - if pkg_resources.resource_exists( - __name__, PlotTemplates.PKG_TEMPLATES_DIR + if ( + importlib_resources.files(__package__) + .joinpath(PlotTemplates.PKG_TEMPLATES_DIR) + .is_dir() ): - return pkg_resources.resource_listdir( - __name__, PlotTemplates.PKG_TEMPLATES_DIR + entries = ( + importlib_resources.files(__package__) + .joinpath(PlotTemplates.PKG_TEMPLATES_DIR) + .iterdir() ) + return [entry.name for entry in entries] return [] @staticmethod @@ -142,9 +147,15 @@ def _load_from_pkg(name): templates = PlotTemplates._get_templates() found = PlotTemplates._find(templates, name) if found: - return pkg_resources.resource_string( - __name__, f"{PlotTemplates.PKG_TEMPLATES_DIR}/{found}" - ).decode("utf-8") + return ( + ( + importlib_resources.files(__package__) + / PlotTemplates.PKG_TEMPLATES_DIR + / found + ) + .read_bytes() + .decode("utf-8") + ) return None def load(self, name: str = None) -> Template: @@ -174,8 +185,11 @@ def init(self): templates = self._get_templates() for template in templates: - content = pkg_resources.resource_string( - __name__, f"{PlotTemplates.PKG_TEMPLATES_DIR}/{template}" + content = ( + importlib_resources.files(__package__) + .joinpath(PlotTemplates.PKG_TEMPLATES_DIR) + .joinpath(template) + .read_text() ) - with open(os.path.join(self.templates_dir, template), "wb") as fd: + with open(os.path.join(self.templates_dir, template), "w") as fd: fd.write(content) diff --git a/tests/conftest.py b/tests/conftest.py index d859191bc3..2f741ba832 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -161,11 +161,13 @@ def pytest_configure(config): @pytest.fixture() def custom_template(tmp_dir, dvc): - import pkg_resources + import importlib_resources - content = pkg_resources.resource_string( - "dvc.repo.plots", "templates/simple.json" - ).decode("utf-8") + content = ( + importlib_resources.files("dvc.repo.plots") + / "templates" + / "simple.json" + ).read_text() template = tmp_dir / "custom_template.json" template.write_text(content) From c1095235de909389086c74f0e8c26ed60bb3ba87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Fri, 8 Oct 2021 20:11:26 +0200 Subject: [PATCH 7/8] fixup --- dvc/repo/plots/template.py | 6 +++++- tests/conftest.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index d604d25ef5..159c62d2a9 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -2,7 +2,11 @@ import os from typing import TYPE_CHECKING, Iterable, Optional -import importlib_resources +try: + import importlib_resources +except ImportError: + import importlib.resources as importlib_resources # type: ignore[no-redef] + from funcy import cached_property from dvc.exceptions import DvcException diff --git a/tests/conftest.py b/tests/conftest.py index 2f741ba832..c9fcd33e7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -161,7 +161,10 @@ def pytest_configure(config): @pytest.fixture() def custom_template(tmp_dir, dvc): - import importlib_resources + try: + import importlib_resources + except ImportError: + import importlib.resources as importlib_resources content = ( importlib_resources.files("dvc.repo.plots") From 4c98f0e5a94983652ea2d604bdd4476af1d2f485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Fri, 8 Oct 2021 22:21:48 +0200 Subject: [PATCH 8/8] fixup --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index ee4b407090..e386250dc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,6 +64,7 @@ install_requires = dataclasses>=0.7; python_version < '3.7' contextvars>=2.1; python_version < '3.7' importlib-metadata>=1.4; python_version < '3.8' + importlib-resources>=5.2.2; python_version < '3.9' flatten_dict>=0.4.1,<1 tabulate>=0.8.7 pygtrie>=2.3.2