Merge 255cdc2 into b78c54c

sacredboard/app/data/datastorage.py

@@ -1,6 +1,7 @@
 """Interfaces for data storage."""
 from sacredboard.app.data.rundao import RunDAO
 from .errors import NotFoundError
+from .filesdao import FilesDAO
 from .metricsdao import MetricsDAO
 
 
@@ -52,6 +53,15 @@ def get_run_dao(self) -> RunDAO:
         raise NotImplementedError(
             "Run Data Access Object must be implemented.")
 
+    def get_files_dao(self) -> FilesDAO:
+        """
+        Return a data access object for files.
+
+        :return: FilesDAO
+        """
+        raise NotImplementedError(
+            "Artifacts Data Access Object must be implemented.")
+
 
 class DummyMetricsDAO(MetricsDAO):
     """Dummy Metrics DAO that does not find any metric."""

sacredboard/app/data/filesdao.py

@@ -0,0 +1,13 @@
+"""Interface for accessing Sacred files."""
+
+
+class FilesDAO:
+    """Interface for accessing files."""
+
+    def get(self, file_id):
+        """
+        Return the file associated with the id.
+
+        :raise NotFoundError when not found
+        """
+        raise NotImplementedError("RunDAO is abstract.")

sacredboard/app/data/pymongo/__init__.py

@@ -1,6 +1,7 @@
 """Module responsible for accessing the MongoDB database."""
 from .genericdao import GenericDAO
 from .metricsdao import MongoMetricsDAO
+from .filesdao import MongoFilesDAO
 from .mongodb import PyMongoDataAccess
 
-__all__ = ("MongoMetricsDAO", "GenericDAO", "PyMongoDataAccess")
+__all__ = ("MongoMetricsDAO", "GenericDAO", "PyMongoDataAccess", "MongoFilesDAO")

sacredboard/app/data/pymongo/filesdao.py

@@ -0,0 +1,35 @@
+"""Module responsible for accessing the Files data in MongoDB."""
+from typing import Union
+
+import bson
+import gridfs
+
+from sacredboard.app.data.pymongo import GenericDAO
+from sacredboard.app.data.filesdao import FilesDAO
+
+
+class MongoFilesDAO(FilesDAO):
+    """Implementation of FilesDAO for MongoDB."""
+
+    def __init__(self, generic_dao: GenericDAO):
+        """
+        Create new Files accessor for MongoDB.
+
+        :param generic_dao: A configured generic MongoDB data access object
+         pointing to an appropriate database.
+        """
+        self.generic_dao = generic_dao
+
+        self._fs = gridfs.GridFS(self.generic_dao._database)
+
+    def get(self, file_id: Union[str, bson.ObjectId]) -> gridfs.GridOut:
+        """
+        Return the file identified by a file_id string.
+        
+        The return value is a file-like object which also has the following attributes:
+        filename: str
+        upload_date: datetime
+        """
+        if isinstance(file_id, str):
+            file_id = bson.ObjectId(file_id)
+        return self._fs.get(file_id)

sacredboard/app/data/pymongo/mongodb.py

@@ -3,7 +3,7 @@
 import pymongo
 
 from sacredboard.app.data.datastorage import Cursor, DataStorage
-from sacredboard.app.data.pymongo import GenericDAO, MongoMetricsDAO
+from sacredboard.app.data.pymongo import GenericDAO, MongoMetricsDAO, MongoFilesDAO
 from sacredboard.app.data.pymongo.rundao import MongoRunDAO
 
 
@@ -102,3 +102,11 @@ def get_run_dao(self):
         :return: RunDAO
         """
         return MongoRunDAO(self._generic_dao, self._collection_name)
+
+    def get_files_dao(self):
+        """
+        Return a data access object for Files.
+
+        :return: RunDAO
+        """
+        return MongoFilesDAO(self._generic_dao)

sacredboard/app/webapi/files.py

@@ -0,0 +1,127 @@
+"""Accessing the files."""
+from enum import Enum
+import io
+import os
+import mimetypes
+import zipfile
+
+from flask import Blueprint, current_app, render_template, send_file, Response
+
+from sacredboard.app.data import NotFoundError
+
+
+files = Blueprint("files", __name__)
+
+
+class _FileType(Enum):
+    ARTIFACT = 1
+    SOURCE = 2
+
+
+_filetype_suffices = {
+    _FileType.ARTIFACT: "artifact",
+    _FileType.SOURCE: "source",
+}
+
+
+def _get_binary_info(binary: bytes):
+    hex_data = ""
+    for i in range(0, 10):
+        if i > 0:
+            hex_data += " "
+        hex_data += hex(binary[i])
+    hex_data += " ..."
+    return "Binary data\nLength: {}\nFirst 10 bytes: {}".format(len(binary), hex_data)
+
+
+def get_file(file_id: str, download):
+    """
+    Get a specific file from GridFS.
+
+    Returns a binary stream response or HTTP 404 if not found.
+    """
+    data = current_app.config["data"]  # type: DataStorage
+    dao = data.get_files_dao()
+    file = dao.get(file_id)
+
+    if download:
+        mime = mimetypes.guess_type(file.filename)[0]
+        if mime is None:
+            # unknown type
+            mime = "binary/octet-stream"
+
+        basename = os.path.basename(file.filename)
+        return send_file(file, mimetype=mime, attachment_filename=basename, as_attachment=True)
+    else:
+        rawdata = file.read()
+        try:
+            text = rawdata.decode('utf-8')
+        except UnicodeDecodeError:
+            # not decodable as utf-8
+            text = _get_binary_info(rawdata)
+        html = render_template("api/file_view.html", content=text)
+        return Response(html)
+
+
+def get_files_zip(run_id: int, filetype: _FileType):
+    """Send all artifacts or sources of a run as ZIP."""
+    data = current_app.config["data"]
+    dao_runs = data.get_run_dao()
+    dao_files = data.get_files_dao()
+    run = dao_runs.get(run_id)
+
+    if filetype == _FileType.ARTIFACT:
+        target_files = run['artifacts']
+    elif filetype == _FileType.SOURCE:
+        target_files = run['experiment']['sources']
+    else:
+        raise Exception("Unknown file type: %s" % filetype)
+
+    memory_file = io.BytesIO()
+    with zipfile.ZipFile(memory_file, 'w') as zf:
+        for f in target_files:
+            # source and artifact files use a different data structure
+            file_id = f['file_id'] if 'file_id' in f else f[1]
+            file = dao_files.get(file_id)
+            data = zipfile.ZipInfo(file.filename, date_time=file.upload_date.timetuple())
+            data.compress_type = zipfile.ZIP_DEFLATED
+            zf.writestr(data, file.read())
+    memory_file.seek(0)
+
+    fn_suffix = _filetype_suffices[filetype]
+    return send_file(memory_file, attachment_filename='run{}_{}.zip'.format(run_id, fn_suffix), as_attachment=True)
+
+
+@files.route("/api/file/<string:file_id>")
+def api_file(file_id):
+    """Download a file."""
+    return get_file(file_id, True)
+
+
+@files.route("/api/fileview/<string:file_id>")
+def api_fileview(file_id):
+    """View a file."""
+    return get_file(file_id, False)
+
+
+@files.route("/api/artifacts/<int:run_id>")
+def api_artifacts(run_id):
+    """Download all artifacts of a run as ZIP."""
+    return get_files_zip(run_id, _FileType.ARTIFACT)
+
+
+@files.route("/api/sources/<int:run_id>")
+def api_sources(run_id):
+    """Download all sources of a run as ZIP."""
+    return get_files_zip(run_id, _FileType.SOURCE)
+
+
+@files.errorhandler(NotFoundError)
+def handle_not_found_error(e):
+    """Handle exception when a metric is not found."""
+    return "Couldn't find resource:\n%s" % e, 404
+
+
+def initialize(app, app_config):
+    """Register the module in Flask."""
+    app.register_blueprint(files)

sacredboard/bootstrap.py

@@ -14,13 +14,13 @@
 from sacredboard.app.config import jinja_filters
 from sacredboard.app.data.filestorage import FileStorage
 from sacredboard.app.data.pymongo import PyMongoDataAccess
-from sacredboard.app.webapi import routes, metrics, runs, proxy
+from sacredboard.app.webapi import routes, metrics, runs, files, proxy
 from sacredboard.app.webapi.wsgi_server import ServerRunner
 
 locale.setlocale(locale.LC_ALL, '')
 app = Flask(__name__)
 server_runner = ServerRunner()
-webapi_modules = [proxy, routes, metrics, runs, jinja_filters, server_runner]
+webapi_modules = [proxy, routes, metrics, runs, files, jinja_filters, server_runner]
 
 
 @click.command()

sacredboard/static/scripts/runs/detailView/component.js

@@ -13,8 +13,9 @@ define(
         "text!./template.html",
         "runs/dictionaryBrowser/component",
         "runs/metricsViewer/component",
+        "runs/filesBrowser/component",
         "jquery"],
-    function (ko, escapeHtml, htmlTemplate, dictionaryBrowser, metricsViewer, $) {
+    function (ko, escapeHtml, htmlTemplate, dictionaryBrowser, metricsViewer, filesBrowser, $) {
         ko.components.register("detail-view", {
             /**
              * Remove run.

sacredboard/static/scripts/runs/detailView/template.html

@@ -46,6 +46,14 @@ <h3>Details for: <!--ko text: run.experiment_name--><!--/ko-->
                     <a data-bind="attr: {href: '#metrics-' + run.id}"
                        data-toggle="pill">Metrics plots</a>
                 </li>
+                <li role="presentation">
+                    <a data-bind="attr: {href: '#artifacts-' + run.id}"
+                       data-toggle="pill">Artifacts</a>
+                </li>
+                <li role="presentation">
+                    <a data-bind="attr: {href: '#sources-' + run.id}"
+                       data-toggle="pill">Sources</a>
+                </li>
                 <li>
                     <button class="btn btn-block btn-danger"
                             data-bind="click: deleteRun">
@@ -146,6 +154,22 @@ <h4>Tensorflow logs</h4>
                     <metrics-viewer params="run: run.object"></metrics-viewer>
                 </div>
             </div>
+            <div data-bind="attr: {id: 'artifacts-' + run.id}"
+                 class="tab-pane table-responsive">
+                <h4>Artifacts</h4>
+                <div class="detail-page-box">
+                    <files-browser
+                            params='files: run.object.artifacts, run_id:run.object._id, all_url:"/api/artifacts/"'></files-browser>
+                </div>
+            </div>
+            <div data-bind="attr: {id: 'sources-' + run.id}"
+                 class="tab-pane table-responsive">
+                <h4>Sources</h4>
+                <div class="detail-page-box">
+                    <files-browser
+                            params='files: run.object.experiment.sources, run_id:run.object._id, all_url:"/api/sources/"'></files-browser>
+                </div>
+            </div>
         </div>
     </div>
 </div>

sacredboard/static/scripts/runs/filesBrowser/component.js

@@ -0,0 +1,37 @@
+"use strict";
+define(["knockout", "text!runs/filesBrowser/template.html"],
+    function (ko, htmlTemplate) {
+        ko.components.register("files-browser", {
+            viewModel: function (params) {
+                var self = this;
+                this.run_id = params.run_id;
+                this.files = params.files;
+                this.url_all = params.all_url;
+                this.files = [];
+
+                // sources files just are arrays. convert to objects here
+                params.files.forEach((element, index, array) => {
+                    if (Array.isArray(element)) {
+                        this.files.push({name: element[0], file_id: element[1]});
+                    }
+                    else {
+                        // just pass through
+                        this.files.push(element);
+                    }
+                });
+
+                this.downloadFile = function (file) {
+                    window.location.href = `api/file/${file.file_id.$oid}`;
+                };
+
+                this.viewFile = function (file) {
+                    window.open(`api/fileview/${file.file_id.$oid}`, "_blank");
+                };
+
+                this.downloadFileAll = function (vm) {
+                    window.location.href = this.url_all + vm.run_id;
+                };
+            },
+            template: htmlTemplate
+        });
+    });

sacredboard/static/scripts/runs/filesBrowser/template.html

@@ -0,0 +1,25 @@
+<!-- ko if: files.length > 0 -->
+<table class="table table-condensed table-hover"
+        style="border-left: groove">
+    <tbody as data-bind="foreach: files">
+        <tr>
+            <td data-bind="text: name"></td>
+            <td>
+                <button class="btn btn-info" data-bind="click: $parent.viewFile">
+                    VIEW
+                </button>
+                <button class="btn btn-primary" data-bind="click: $parent.downloadFile">
+                    DOWNLOAD
+                </button>
+            </td>
+        </tr>
+    </tbody>
+</table>
+<button class="btn btn-primary" data-bind="click: downloadFileAll">
+    DOWNLOAD ALL
+</button>
+<!-- /ko -->
+<!-- ko ifnot: files.length > 0 -->
+No files
+<!-- /ko -->
+

sacredboard/templates/api/file_view.html

@@ -0,0 +1,6 @@
+<style type="text/css">
+    div {
+        white-space: pre-wrap;
+    }
+</style>
+<div>{{ content }}</div>

sacredboard/templates/api/runs.js

@@ -15,6 +15,22 @@
         "heartbeat": {{run.heartbeat | default | format_datetime | tojson}},
         "heartbeat_diff": {{run.heartbeat | default | timediff | tojson}},
         "hostname": {{run.host.hostname | default | tojson}},
+        "artifacts": [
+        {%- for artifact in run.artifacts -%}
+            {"filename": {{artifact.name | default | tojson}}, "file_id" : {{artifact.file_id | default | string | tojson}}}
+            {%- if not loop.last -%}
+                ,
+            {% endif %}
+        {% endfor %}
+        ],
+        "sources": [
+        {%- for source in run.experiment.sources -%}
+            {"filename": {{source[0] | default | tojson}}, "file_id" : {{source[1] | default | string | tojson}}}
+            {%- if not loop.last -%}
+                ,
+            {% endif %}
+        {% endfor %}
+        ],
         {# commented out: "captured_out_last_line": {{run.captured_out | default | last_line | tojson}}, #}
         "result":{{run.result | default | dump_json }}
         {%- if full_object -%},