Add interactive training editor. Closes #1808 (#1816)

* WIP #1808 Add generated files for TrainKeras

* WIP Add python code for TrainKeras editor (plotting losses)

* Use plotly matplotlib backend, fix example model

* Add dashboard (and svelte build commands)

* Add rollup config for svelte

* Add fn parameter config generation

* Fix update arch. Add loss config

* Add support for reductions in loss fns

* Remove learning rate (included as optimizer option)

* Remove lr (optimizer config instead)

* Update plotly path

* Add getArchitectureCode to widget

* debugging code

* Add generated architecture code to the model

* Set loss, optimizer and update plots in real time

* Add Train button

* add help links for loss, optimizer

* Use validation when training model

* Add training data field. Ensure correct order when killing current task

* Add training data selector

* Add sections for config

* Add validation loss to plots

* Show trained models

* Prompt storage info on save

* Update model state on upload

* Fix plot updates, python friendly artifact names

* Return storage config info

* Add rollup deps

* Use deepforge serializer for models

* Update training button. Upload artifact on save

* Use JSON importer for saving snapshot

* Add JSON importer

* Fix model saving

* Show info on click for errors

* Update JSON Importer

* Don't pass jquery to initialize

* Update pipeline seed to have TrainKeras operation...

* Use correct filepath when saving artifact

* Fix svelte warnings

* Add build files to codeclimate ignore

* Add more ignore paths

* Clean up the code (treat artifacts as the main territory)

* Refactor line collector and add tests

* Fixed error with getObjectDescriptor

* Remove line collector; clear save errors

.codeclimate.yml

@@ -41,3 +41,6 @@ exclude_paths:
 - src/plugins/GenerateJob/templates/utils.build.js
 - src/visualizers/widgets/TensorPlotter/lib/
 - src/visualizers/widgets/TensorPlotter/styles/simple-grid.min.css
+- src/visualizers/widgets/TrainKeras/build
+- src/visualizers/panels/TrainKeras/JSONImporter.js
+- src/visualizers/panels/TrainKeras/changeset.js

package.json

@@ -9,7 +9,9 @@
   },
   "scripts": {
     "start": "node ./bin/deepforge start",
-    "postinstall": "node utils/reinstall-extensions.js && node utils/build-job-utils.js && ./bin/deepforge create-env",
+    "postinstall": "node utils/reinstall-extensions.js && npm run build && ./bin/deepforge create-env",
+    "build": "node utils/build-job-utils.js && npm run build-svelte",
+    "build-svelte": "rollup -c utils/svelte/rollup.config.js",
     "start-dev": "NODE_ENV=dev node ./bin/deepforge start",
     "start-authenticated": "NODE_ENV=production ./bin/deepforge start",
     "server": "node ./bin/deepforge start --server",
@@ -61,12 +63,19 @@
     "ws": "^7.2.5"
   },
   "devDependencies": {
+    "@rollup/plugin-commonjs": "^12.0.0",
+    "@rollup/plugin-node-resolve": "^8.4.0",
     "chai": "^3.0.0",
     "deepforge-worker": "git+https://github.com/deepforge-dev/worker.git",
     "jszip": "^2.5.0",
     "mocha": "^2.2.5",
     "mockery": "^1.7.0",
     "nodemon": "^1.9.2",
-    "skulpt": "^0.11.1"
+    "skulpt": "^0.11.1",
+    "rollup": "^2.3.4",
+    "rollup-plugin-livereload": "^1.0.0",
+    "rollup-plugin-svelte": "^5.0.3",
+    "rollup-plugin-terser": "^5.1.2",
+    "svelte": "^3.0.0"
   }
 }

src/common/compute/line-collector.js

@@ -0,0 +1,37 @@
+/*globals define*/
+define([], function() {
+    class LineCollector {
+        constructor() {
+            this.currentLine = '';
+            this.handler = null;
+        }
+
+        on(fn) {
+            this.handler = fn;
+        }
+
+        receive(data) {
+            const text = data.toString();
+            const newLineIndex = text.indexOf('\n');
+            let fragment;
+            if (newLineIndex > -1) {
+                const line = this.currentLine + text.substring(0, newLineIndex);
+                this.handler(line);
+                fragment = text.substring(newLineIndex + 1);
+                this.currentLine = '';
+            } else {
+                fragment = text;
+            }
+            this.currentLine += fragment;
+        }
+
+        flush() {
+            if (this.currentLine) {
+                this.handler(this.currentLine);
+                this.currentLine = '';
+            }
+        }
+    }
+
+    return LineCollector;
+});

src/seeds/pipeline/releases.jsonl

@@ -2,3 +2,4 @@
 {"version":"0.21.0","changelog":"Add provenance to metadata (via WithProvenance mixin)"}
 {"version":"0.21.1","changelog":"Update Inheritance of Subgraph, Line, Images, ScatterPoints etc.. nodes"}
 {"version":"0.22.0","changelog":"Incorporate PlotlyJSON into Graph meta node"}
+{"version":"0.23.0","changelog":"Add TrainKeras implicit operation"}

src/visualizers/Visualizers.json

@@ -155,10 +155,16 @@
     "panel": "panels/InteractiveExplorer/InteractiveExplorerPanel",
     "DEBUG_ONLY": false
   },
+  {
+    "id": "TrainKeras",
+    "title": "TrainKeras",
+    "panel": "panels/TrainKeras/TrainKerasPanel",
+    "DEBUG_ONLY": false
+  },
   {
     "id": "PlotlyGraph",
     "title": "PlotlyGraph",
     "panel": "panels/PlotlyGraph/PlotlyGraphPanel",
     "DEBUG_ONLY": false
   }
-]
+]

src/visualizers/panels/InteractiveEditor/InteractiveEditorControl.js

@@ -70,7 +70,7 @@ define([
             return deferred.promise;
         }
 
-        selectedObjectChanged (nodeId) {
+        async selectedObjectChanged (nodeId) {
             const desc = this.getObjectDescriptor(nodeId);
 
             this._logger.debug('activeObject nodeId \'' + nodeId + '\'');
@@ -82,7 +82,7 @@ define([
             this._currentNodeId = nodeId;
 
             if (typeof this._currentNodeId === 'string') {
-                const territory = this.getTerritory(nodeId);
+                const territory = await this.getTerritory(nodeId);
                 this._widget.setTitle(desc.name.toUpperCase());
 
                 this._territoryId = this.client