feat: Artifacts tab should not load all artifacts by default and shou…

…ld improve performance

Download / Download All should stream data from server and not crash the browser

BREAKING CHANGE: File download, download all and file preview APIs have changed

fixes #71

package.json

@@ -19,11 +19,13 @@
     "url": "https://github.com/vivekratnavel/omniboard.git"
   },
   "dependencies": {
+    "archiver": "^3.0.0",
     "body-parser": "~1.18.2",
     "cross-env": "^5.2.0",
     "express": "~4.15.5",
     "express-paginate": "^0.3.0",
     "express-restify-mongoose": "^4.3.0",
+    "gridfs-stream": "^1.1.1",
     "method-override": "^2.3.10",
     "mongodb": "^3.0.1",
     "mongoose": "^5.3.3",
@@ -32,6 +34,7 @@
   },
   "scripts": {
     "start": "babel-node server",
+    "debug": "nodemon --ignore './web/' server --exec babel-node --inspect",
     "server": "nodemon --ignore './web/' server --exec babel-node",
     "client": "cd web && npm run start",
     "clean:server": "rm -rf ./dist/*",

server/app.js

@@ -1,4 +1,4 @@
-import database from './config/database';
+import {databaseConn, gfs} from './config/database';
 import express from 'express';
 import * as path from 'path';
 import morgan from 'morgan';
@@ -11,6 +11,7 @@ import OmniboardColumnsModel from './models/omniboard.columns';
 import OmniboardConfigColumnsModel from './models/omniboard.config.columns';
 import FilesModel from './models/fs.files';
 import ChunksModel from './models/fs.chunks';
+import archiver from 'archiver';
 
 const app = express();
 const router = express.Router();
@@ -60,9 +61,109 @@ router.get('/api/v1/files/:id', function(req, res) {
   });
 });
 
+router.get('/api/v1/files/download/:id/:fileName', function(req, res) {
+  // Read file as stream from Mongo GridFS
+  const readStream = gfs.createReadStream({
+    _id: req.params.id
+  });
+  //error handling, e.g. file does not exist
+  readStream.on('error', function (err) {
+    console.error('An error occurred: ', err);
+    throw err;
+  });
+
+  const fileName = req.params.fileName;
+  res.contentType(fileName);
+  res.set({
+    'Content-Disposition': 'attachment; filename=' + fileName
+  });
+
+  // Pipe the file stream to http response
+  readStream.pipe(res);
+});
+
+router.get('/api/v1/files/downloadAll/:runId/:fileType', function(req, res) {
+  const FILE_TYPE = {
+    SOURCE_FILES: 'source_files',
+    ARTIFACTS: 'artifacts'
+  };
+  const allowedTypes = [
+    FILE_TYPE.SOURCE_FILES,
+    FILE_TYPE.ARTIFACTS
+  ];
+  const fileType = req.params.fileType;
+  const runId = req.params.runId;
+  if (!allowedTypes.includes(fileType)) {
+    res.status(400).json({message: 'Error: Invalid input for fileType.'});
+  } else {
+    RunsModel.findById(req.params.runId).exec(function(err, result) {
+      if (err) throw (err);
+      let files = [];
+      if (fileType === FILE_TYPE.SOURCE_FILES) {
+        if (result && result.experiment && result.experiment.sources) {
+          files = result.experiment.sources.map(source => {
+            return {
+              name: source[0],
+              file_id: source[1]
+            }
+          });
+        } else {
+          res.status(500).json({message: 'Error: Unable to fetch source files for runId: ' + runId});
+        }
+      } else {
+        // fileType: artifacts
+        files = result.artifacts;
+      }
+      const archive = archiver('zip', {
+        zlib: { level: 5 } // Sets the compression level.
+      });
+      const fileName = `${fileType}-${runId}.zip`; // ex: source-files-1.zip
+      const dirName = `${fileType}-${runId}`; // ex: source-files-1
+      archive.on('error', function(err) {
+        console.error('An error occurred: ', err);
+        res.status(500);
+        throw err;
+      });
+      files.forEach(function(file) {
+        const readStream = gfs.createReadStream({
+          _id: file.file_id
+        });
+        //error handling, e.g. file does not exist
+        readStream.on('error', function (err) {
+          console.error('An error occurred: ', err);
+          res.status(500);
+          throw err;
+        });
+        // add file to archive
+        archive.append(readStream, {name: file.name, prefix: dirName});
+      });
+      archive.finalize();
+      res.set({
+        'Content-Disposition': 'attachment; filename=' + fileName
+      });
+      archive.pipe(res);
+    });
+  }
+});
+
+router.get('/api/v1/files/preview/:fileId', function(req, res) {
+  // Read file as stream from Mongo GridFS
+  const readStream = gfs.createReadStream({
+    _id: req.params.fileId
+  });
+  //error handling, e.g. file does not exist
+  readStream.on('error', function (err) {
+    console.error('An error occurred: ', err);
+    throw err;
+  });
+
+  // Pipe the file stream to http response
+  readStream.pipe(res);
+});
+
 router.get('/api/v1/database', function(req, res) {
-  if (database && database.name) {
-    res.json({name: database.name});
+  if (databaseConn && databaseConn.name) {
+    res.json({name: databaseConn.name});
   } else {
     res.status(500).json({message: 'An unknown error occurred'})
   }

server/config/database.js

@@ -1,5 +1,6 @@
 import mongoose from 'mongoose';
 import yargs from 'yargs';
+import Grid from 'gridfs-stream';
 
 const argv = yargs.argv;
 
@@ -32,6 +33,7 @@ if ('m' in argv) {
 const mongoOptions = { auto_reconnect: true, useNewUrlParser: true };
 const reconnectTries = 2;
 let counter = 0;
+let gfs = null;
 const createConnection = function(mongodbURI, mongoOptions) {
   const db = mongoose.createConnection(mongodbURI, mongoOptions);
 
@@ -66,9 +68,11 @@ const createConnection = function(mongodbURI, mongoOptions) {
 
   db.once('open', function() {
     console.log(`Connection to ${mongodbURI} established successfully!`);
+    gfs = Grid(db.db, mongoose.mongo);
   });
 
   return db;
 };
+const databaseConn = createConnection(mongodbURI, mongoOptions);
 
-export default createConnection(mongodbURI, mongoOptions);
+export {databaseConn, gfs};

server/models/fs.chunks.js

@@ -1,5 +1,5 @@
 import mongoose from 'mongoose';
-import db from '../config/database';
+import {databaseConn} from '../config/database';
 
 const Schema = mongoose.Schema;
 mongoose.Promise = Promise;
@@ -13,4 +13,4 @@ export const ChunksSchema = new Schema({
   strict: false
 });
 
-export default db.model('fs.chunks', ChunksSchema);
+export default databaseConn.model('fs.chunks', ChunksSchema);

server/models/fs.files.js

@@ -1,5 +1,5 @@
 import mongoose from 'mongoose';
-import db from '../config/database';
+import {databaseConn} from '../config/database';
 
 const Schema = mongoose.Schema;
 mongoose.Promise = Promise;
@@ -17,4 +17,4 @@ FilesSchema.virtual('chunk', {
   foreignField: 'files_id'
 });
 
-export default db.model('fs.files', FilesSchema);
+export default databaseConn.model('fs.files', FilesSchema);

server/models/metrics.js

@@ -1,5 +1,5 @@
 import mongoose from 'mongoose';
-import db from '../config/database';
+import {databaseConn} from '../config/database';
 
 const Schema = mongoose.Schema;
 mongoose.Promise = Promise;
@@ -11,4 +11,4 @@ export const MetricsSchema = new Schema({
   strict: false
 });
 
-export default db.model('metrics', MetricsSchema);
+export default databaseConn.model('metrics', MetricsSchema);

server/models/omniboard.columns.js

@@ -1,5 +1,5 @@
 import mongoose from 'mongoose';
-import db from '../config/database';
+import {databaseConn} from '../config/database';
 
 const Schema = mongoose.Schema;
 mongoose.Promise = Promise;
@@ -12,4 +12,4 @@ export const OmniboardColumnsSchema = new Schema({
   strict: false
 });
 
-export default db.model('omniboard.columns', OmniboardColumnsSchema);
+export default databaseConn.model('omniboard.columns', OmniboardColumnsSchema);

server/models/omniboard.config.columns.js

@@ -1,5 +1,5 @@
 import mongoose from 'mongoose';
-import db from '../config/database';
+import {databaseConn} from '../config/database';
 
 const Schema = mongoose.Schema;
 mongoose.Promise = Promise;
@@ -11,4 +11,4 @@ export const OmniboardConfigColumnsSchema = new Schema({
   strict: false
 });
 
-export default db.model('omniboard.config.columns', OmniboardConfigColumnsSchema);
+export default databaseConn.model('omniboard.config.columns', OmniboardConfigColumnsSchema);

server/models/runs.js

@@ -1,22 +1,23 @@
 import mongoose from 'mongoose';
-import db from '../config/database';
+import {databaseConn} from '../config/database';
 
 const Schema = mongoose.Schema;
 mongoose.Promise = Promise;
 
 export const RunsSchema = new Schema({
-    _id: {type: Number},
-    heartbeat: {type: Date},
-    meta: {},
-    config: {},
-    experiment: {},
-    start_time: {type: Date},
-    omniboard: {
-        tags: [
-          { type: String }
-        ],
-        notes: {type: String}
-    }
+  _id: {type: Number},
+  heartbeat: {type: Date},
+  meta: {},
+  config: {},
+  experiment: {},
+  start_time: {type: Date},
+  omniboard: {
+      tags: [
+        { type: String }
+      ],
+      notes: {type: String}
+  },
+  artifacts: []
 }, {
     strict: false
 });
@@ -27,4 +28,4 @@ RunsSchema.virtual('metrics', {
   foreignField: 'run_id'
 });
 
-export default db.model('runs', RunsSchema);
+export default databaseConn.model('runs', RunsSchema);

web/config/jest/jestSetup.js

@@ -9,6 +9,7 @@ Enzyme.configure({ adapter: new Adapter() });
 global.shallow = shallow;
 global.render = render;
 global.mount = mount;
+global.fetch = require('jest-fetch-mock');
 
 // From Stack Overflow
 // https://stackoverflow.com/questions/37408834/testing-with-reacts-jest-and-enzyme-when-simulated-clicks-call-a-function-that

web/package.json

@@ -41,7 +41,6 @@
     "html-webpack-plugin": "2.29.0",
     "ify-loader": "^1.1.0",
     "jest": "20.0.4",
-    "jszip": "^3.1.5",
     "ms": "^2.1.1",
     "object-assign": "4.1.1",
     "plotly.js": "^1.33.1",
@@ -60,6 +59,7 @@
     "react-dev-utils": "^4.2.1",
     "react-dom": "^16.2.0",
     "react-json-view": "^1.16.0",
+    "react-list": "^0.8.11",
     "react-localstorage": "^0.3.1",
     "react-mixin": "2",
     "react-plotly.js": "^1.3.0",
@@ -149,6 +149,7 @@
     "enzyme": "^3.4.4",
     "enzyme-adapter-react-16": "^1.2.0",
     "enzyme-to-json": "^3.3.4",
+    "jest-fetch-mock": "^2.1.1",
     "jest-mock-axios": "^2.1.11",
     "node-sass": "^4.7.2",
     "react-hot-loader": "^4.0.0-beta.21",

web/src/components/DrillDownView/__snapshots__/drillDownView.test.js.snap

@@ -9,6 +9,7 @@ exports[`DrillDownView should render correctly 1`] = `
       "selectedNavTab",
     ]
   }
+  status="COMPLETED"
   width={600}
 >
   <div

web/src/components/DrillDownView/drillDownView.test.js

@@ -55,7 +55,7 @@ describe('DrillDownView', () => {
 
   beforeEach(() => {
     wrapper = mount(
-      <DrillDownView height={500} runId={200} width={600}/>
+      <DrillDownView height={500} runId={200} width={600} status={STATUS.COMPLETED}/>
     );
   });
 
@@ -172,13 +172,13 @@ describe('DrillDownView', () => {
     wrapper.unmount();
     mockAxios.reset();
     wrapper = mount(
-      <DrillDownView height={500} width={600}/>
+      <DrillDownView height={500} width={600} runId={0} status={STATUS.COMPLETED}/>
     );
 
     expect(mockAxios.get).not.toHaveBeenCalled();
     wrapper.unmount();
     wrapper = mount(
-      <DrillDownView height={500} runId={1} width={600}/>
+      <DrillDownView height={500} runId={1} width={600} status={STATUS.COMPLETED}/>
     );
 
     expect(mockAxios.get).toHaveBeenCalledTimes(2);