plots: introduce support for images

dvc/command/live.py

@@ -1,5 +1,5 @@
 import argparse
-import os
+from pathlib import Path
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
@@ -13,10 +13,11 @@ def _run(self, target, revs=None):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         if plots:
-            html_path = self.args.target + ".html"
-            self.repo.plots.write_html(html_path, plots, metrics)
+            html_path = Path.cwd() / (self.args.target + "_html")
+            from dvc.render.utils import render
 
-            ui.write("\nfile://", os.path.abspath(html_path), sep="")
+            index_path = render(self.repo, plots, metrics, html_path)
+            ui.write(index_path.as_uri())
             return 0
         return 1
 

dvc/command/plots.py

@@ -4,6 +4,7 @@
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.render.utils import find_vega, render
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -35,43 +36,50 @@ def run(self):
                 return 1
 
         try:
-            plots = self._func(targets=self.args.targets, props=self._props())
+            plots_data = self._func(
+                targets=self.args.targets, props=self._props()
+            )
+
+            if not plots_data:
+                ui.error_write(
+                    "No plots were loaded, "
+                    "visualization file will not be created."
+                )
 
             if self.args.show_vega:
                 target = self.args.targets[0]
-                ui.write(plots[target])
+                plot_json = find_vega(self.repo, plots_data, target)
+                if plot_json:
+                    ui.write(plot_json)
                 return 0
 
-        except DvcException:
-            logger.exception("")
-            return 1
-
-        if plots:
-            rel: str = self.args.out or "plots.html"
+            rel: str = self.args.out or "dvc_plots"
             path: Path = (Path.cwd() / rel).resolve()
-            self.repo.plots.write_html(
-                path, plots=plots, html_template_path=self.args.html_template
+            index_path = render(
+                self.repo,
+                plots_data,
+                path=path,
+                html_template_path=self.args.html_template,
             )
 
-            assert (
-                path.is_absolute()
-            )  # as_uri throws ValueError if not absolute
-            url = path.as_uri()
+            assert index_path.is_absolute()
+            url = index_path.as_uri()
             ui.write(url)
+
             if self.args.open:
                 import webbrowser
 
-                opened = webbrowser.open(rel)
+                opened = webbrowser.open(index_path)
                 if not opened:
                     ui.error_write(
                         "Failed to open. Please try opening it manually."
                     )
                     return 1
-        else:
-            ui.error_write(
-                "No plots were loaded, visualization file will not be created."
-            )
-        return 0
+            return 0
+
+        except DvcException:
+            logger.exception("")
+            return 1
 
 
 class CmdPlotsShow(CmdPlots):

dvc/render/__init__.py

@@ -0,0 +1,33 @@
+import logging
+from typing import TYPE_CHECKING, Dict
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+logger = logging.getLogger(__name__)
+
+
+class Renderer:
+    def __init__(self, data: Dict):
+        self.data = data
+
+        from dvc.render.utils import get_files
+
+        files = get_files(self.data)
+
+        # we assume comparison of same file between revisions for now
+        assert len(files) == 1
+        self.filename = files.pop()
+
+    def _convert(self, path: "StrPath"):
+        raise NotImplementedError
+
+    @property
+    def DIV(self):
+        raise NotImplementedError
+
+    def generate_html(self, path: "StrPath"):
+        """this method might edit content of path"""
+        partial = self._convert(path)
+        div_id = f"plot_{self.filename.replace('.', '_').replace('/', '_')}"
+        return self.DIV.format(id=div_id, partial=partial)

dvc/utils/html.py → dvc/render/html.py

@@ -1,7 +1,12 @@
-from typing import Dict, List, Optional
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from dvc.exceptions import DvcException
-from dvc.types import StrPath
+
+if TYPE_CHECKING:
+    from dvc.render import Renderer
+    from dvc.types import StrPath
 
 PAGE_HTML = """<!DOCTYPE html>
 <html>
@@ -16,12 +21,6 @@
 </body>
 </html>"""
 
-VEGA_DIV_HTML = """<div id = "{id}"></div>
-<script type = "text/javascript">
-    var spec = {vega_json};
-    vegaEmbed('#{id}', spec);
-</script>"""
-
 
 class MissingPlaceholderError(DvcException):
     def __init__(self, placeholder):
@@ -56,15 +55,6 @@ def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML":
         self.elements.append(tabulate.tabulate(rows, header, tablefmt="html"))
         return self
 
-    def with_plots(self, plots: Dict[str, Dict]) -> "HTML":
-        self.elements.extend(
-            [
-                VEGA_DIV_HTML.format(id=f"plot{i}", vega_json=plot)
-                for i, plot in enumerate(plots.values())
-            ]
-        )
-        return self
-
     def with_element(self, html: str) -> "HTML":
         self.elements.append(html)
         return self
@@ -75,11 +65,14 @@ def embed(self) -> str:
 
 
 def write(
-    path: StrPath,
-    plots: Dict[str, Dict],
+    path: "StrPath",
+    renderers: List["Renderer"],
     metrics: Optional[Dict[str, Dict]] = None,
-    template_path: Optional[StrPath] = None,
+    template_path: Optional["StrPath"] = None,
 ):
+
+    os.makedirs(path, exist_ok=True)
+
     page_html = None
     if template_path:
         with open(template_path) as fobj:
@@ -90,7 +83,11 @@ def write(
         document.with_metrics(metrics)
         document.with_element("<br>")
 
-    document.with_plots(plots)
+    for renderer in renderers:
+        document.with_element(renderer.generate_html(path))
+
+    index = Path(os.path.join(path, "index.html"))
 
-    with open(path, "w") as fd:
+    with open(index, "w") as fd:
         fd.write(document.embed())
+    return index

dvc/render/image.py

@@ -0,0 +1,64 @@
+import os
+from typing import TYPE_CHECKING
+
+from dvc.render import Renderer
+from dvc.render.utils import get_files
+from dvc.utils import relpath
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
+class ImageRenderer(Renderer):
+    DIV = """
+        <div
+            id="{id}"
+            style="border: 1px solid;">
+            {partial}
+        </div>"""
+
+    def _write_image(
+        self,
+        page_dir_path: "StrPath",
+        revision: str,
+        filename: str,
+        image_data: bytes,
+    ):
+        static = os.path.join(page_dir_path, "static")
+        os.makedirs(static, exist_ok=True)
+
+        img_path = os.path.join(
+            static, f"{revision}_{filename.replace(os.sep, '_')}"
+        )
+        rel_img_path = relpath(img_path, page_dir_path)
+        with open(img_path, "wb") as fd:
+            fd.write(image_data)
+        return """
+        <div>
+            <p>{title}</p>
+            <img src="{src}">
+        </div>""".format(
+            title=revision, src=rel_img_path
+        )
+
+    def _convert(self, path: "StrPath"):
+        div_content = []
+        for rev, rev_data in self.data.items():
+            if "data" in rev_data:
+                for file, file_data in rev_data.get("data", {}).items():
+                    if "data" in file_data:
+                        div_content.append(
+                            self._write_image(
+                                path, rev, file, file_data["data"]
+                            )
+                        )
+        if div_content:
+            div_content.insert(0, f"<p>{self.filename}</p>")
+            return "\n".join(div_content)
+        return ""
+
+    @staticmethod
+    def matches(data):
+        files = get_files(data)
+        extensions = set(map(lambda f: os.path.splitext(f)[1], files))
+        return extensions.issubset({".jpg", ".jpeg", ".gif", ".png"})

dvc/render/utils.py

@@ -0,0 +1,80 @@
+import os.path
+from typing import Dict, List, Set
+
+
+def get_files(data: Dict) -> Set:
+    files = set()
+    for rev in data.keys():
+        for file in data[rev].get("data", {}).keys():
+            files.add(file)
+    return files
+
+
+def group_by_filename(plots_data: Dict) -> List[Dict]:
+    # TODO use dpath.util.search once
+    #  https://github.com/dpath-maintainers/dpath-python/issues/147 is released
+    #  now cannot search when errors are present in data
+    files = get_files(plots_data)
+    grouped = []
+
+    for file in files:
+        tmp: Dict = {}
+        for revision, revision_data in plots_data.items():
+            if file in revision_data.get("data", {}):
+                if "data" not in tmp:
+                    tmp[revision] = {"data": {}}
+                tmp[revision]["data"].update(
+                    {file: revision_data["data"][file]}
+                )
+        grouped.append(tmp)
+
+    return grouped
+
+
+def find_vega(repo, plots_data, target):
+    # TODO same as group_by_filename
+    grouped = group_by_filename(plots_data)
+    found = None
+    for plot_group in grouped:
+        files = get_files(plot_group)
+        assert len(files) == 1
+        file = files.pop()
+        if file == target:
+            found = plot_group
+            break
+
+    from dvc.render.vega import VegaRenderer
+
+    if found and VegaRenderer.matches(found):
+        return VegaRenderer(found, repo.plots.templates).get_vega()
+    return ""
+
+
+def match_renderers(plots_data, templates):
+    from dvc.render.image import ImageRenderer
+    from dvc.render.vega import VegaRenderer
+
+    renderers = []
+    for g in group_by_filename(plots_data):
+        if VegaRenderer.matches(g):
+            renderers.append(VegaRenderer(g, templates))
+        if ImageRenderer.matches(g):
+            renderers.append(ImageRenderer(g))
+    return renderers
+
+
+def render(repo, plots_data, metrics=None, path=None, html_template_path=None):
+    # TODO we could probably remove repo usages (here and in VegaRenderer)
+    renderers = match_renderers(plots_data, repo.plots.templates)
+    if not html_template_path:
+        html_template_path = repo.config.get("plots", {}).get(
+            "html_template", None
+        )
+        if html_template_path and not os.path.isabs(html_template_path):
+            html_template_path = os.path.join(repo.dvc_dir, html_template_path)
+
+    from dvc.render.html import write
+
+    return write(
+        path, renderers, metrics=metrics, template_path=html_template_path
+    )