Dm 4567: Table (JS): large table handling

.eslintrc

@@ -13,6 +13,7 @@
     "classes" : true,
     "defaultParams" : true,
     "destructuring" : true,
+    "restParams" : true,
     "forOf": true,
     "jsx": true,
     "modules" : true,

package.json

@@ -15,7 +15,7 @@
     "icepick"   : "0.2.0",
     "isomorphic-fetch": "2.2.1",
     "local-storage" : "1.4.2",
-    "lodash"    : "3.10.1",
+    "lodash"    : "4.0.1",
     "numeral"   : "1.5.3",
     "react"     : "0.14.7",
     "react-dom" : "0.14.7",
@@ -34,7 +34,7 @@
     "uniq"      : "1.0.1",
     "validator" : "4.5.0",
     "whatwg-fetch": "0.10.1",
-    "react-color": "1.3.3"
+    "react-color": "1.3.6"
   },
   "devDependencies": {
     "chai": "^2.3.0",

src/fftools/js/fftools.js

@@ -2,7 +2,7 @@
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
 
-import get from 'lodash/object/get';
+import {get} from 'lodash';
 
 import React from 'react';
 import ReactDOM from 'react-dom';
@@ -15,24 +15,23 @@ import ResultsPanel from 'firefly/ui/ResultsPanel.jsx';
 import FormPanel from 'firefly/ui/FormPanel.jsx';
 import TestImagePanel from 'firefly/visualize/ui/TestImagePanel.jsx';
 import {ExpandedModeDisplay} from 'firefly/visualize/iv/ExpandedModeDisplay.jsx';
-import TablePanel from 'firefly/tables/ui/TablePanel.jsx';
+import {TablePanel} from 'firefly/tables/ui/TablePanel.jsx';
 import Validate from 'firefly/util/Validate.js';
-import TblUtil from 'firefly/tables/TableUtil.js';
+import * as TblUtil from 'firefly/tables/TableUtil.js';
 import HistogramTableViewPanel from 'firefly/visualize/HistogramTableViewPanel.jsx';
 import XYPlotTableViewPanel from 'firefly/visualize/XYPlotTableViewPanel.jsx';
 import {VisHeader} from 'firefly/visualize/ui/VisHeader.jsx';
 import {VisToolbar} from 'firefly/visualize/ui/VisToolbar.jsx';
 
 import FieldGroup from 'firefly/ui/FieldGroup.jsx';
-import CompleteButton from 'firefly/ui/CompleteButton.jsx';
 import ValidationField from 'firefly/ui/ValidationField.jsx';
 
 import {TableRequest} from 'firefly/tables/TableRequest.js';
 
 import TableStatsCntlr from 'firefly/visualize/TableStatsCntlr.js';
 import HistogramCntlr from 'firefly/visualize/HistogramCntlr.js';
 import XYPlotCntlr from 'firefly/visualize/XYPlotCntlr.js';
-import TablesCntlr from 'firefly/tables/TablesCntlr.js';
+import * as TablesCntlr from 'firefly/tables/TablesCntlr.js';
 import {getRootURL} from 'firefly/util/BrowserUtil.js';
 import {download} from 'firefly/util/WebUtil.js';
 
@@ -79,7 +78,6 @@ const App = React.createClass({
                 id:'IpacTableFromSource',
                 source: request.srcTable,
                 tbl_id:  newActiveTblId(),
-                pageSize: 50,
                 filters: request.filters
             });
 
@@ -101,7 +99,6 @@ const App = React.createClass({
         const tblId = table ? table.tbl_id : undefined;
         const highlightedRow = table ? table.highlightedRow : undefined;
 
-        const v = get(this.props, 'appData.props.version') || 'unknown';
         if (!appData.isReady) {
             return (
                 <div>
@@ -155,11 +152,11 @@ const App = React.createClass({
                     <main>
                         <ResultsPanel title={title}
                             imagePlot = {appData.layoutInfo.mode==='expand' ?
-                                             <ExpandedModeDisplay forceExpandedMode={true}/> :
-                                             <TestImagePanel /> }
+                                             <ExpandedModeDisplay   key='results-plots-expanded' forceExpandedMode={true}/> :
+                                             <TestImagePanel key='results-plots'/> }
                             visToolbar = {<VisToolbar/>}
-                            xyPlot = {<XYPlotTableViewPanel tblStatsData={tblStatsData} tblPlotData={xyPlotData} tblId={tblId} highlightedRow={highlightedRow}/>
}
-                            tables = { <TablePanel tableModel={table} selectable={true}/> }
+                            xyPlot = {<XYPlotTableViewPanel  key='results-xyplots' tblStatsData={tblStatsData} tblPlotData={xyPlotData} tblId={tblId} highlightedRow={highlightedRow}/>
}
+                            tables = {<TablePanel key='results-tables' tbl_id={tblId} selectable={true}/> }
                             layoutInfo = { appData.layoutInfo }
                         />
                     </main>

src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupFilter.java

@@ -106,18 +106,6 @@ void start() throws IOException {
             attributes.add(new DataGroup.Attribute("col." + DataGroup.ROWID_NAME + ".Visibility", "hidden"));
         }
 
-        // combine meta with file attributes
-        if (meta == null) {
-            meta = new HashMap<>(attributes.size());
-        } else {
-            for (String k : meta.keySet()) {
-                attributes.add(new DataGroup.Attribute(k, meta.get(k)));
-            }
-        }
-        for (DataGroup.Attribute at : attributes) {
-            meta.put(at.getKey(), at.getValue());
-        }
-
         DataGroup dg = new DataGroup(null, headers);
 
         boolean needToWriteHeader = true;
@@ -189,7 +177,7 @@ public void run() {
                             if (CollectionUtil.matches(rowIdx, row, filters)) {
                                 row.setRowIdx(rowIdx);
                                 IpacTableUtil.writeRow(writer, headers, row);
-                                if (++rowsFound % minPrefetchSize == 0) {
+                                if (++rowsFound % 5000 == 0) {
                                     IpacTableUtil.sendLoadStatusEvents(meta, outf, rowsFound, DataGroupPart.State.INPROGRESS);
                                 }
                             }

src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java

@@ -23,7 +23,7 @@
  * @version $Id: DataGroupReader.java,v 1.13 2012/11/05 18:59:59 loi Exp $
  */
 public class DataGroupReader {
-    public static final int MIN_PREFETCH_SIZE = AppProperties.getIntProperty("IpacTable.min.prefetch.size", 5000);
+    public static final int MIN_PREFETCH_SIZE = AppProperties.getIntProperty("IpacTable.min.prefetch.size", 500);
     public static final String LINE_SEP = System.getProperty("line.separator");
     private static final Logger.LoggerImpl logger = Logger.getLogger();
 

src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupWriter.java

@@ -130,7 +130,7 @@ public void run() {
                         while(itr.hasNext()) {
                             DataObject row = itr.next();
                             IpacTableUtil.writeRow(writer, headers, row);
-                            if (++rowCount % minPrefetchSize == 0) {
+                            if (++rowCount % 5000 == 0) {
                                 IpacTableUtil.sendLoadStatusEvents(meta, outf, rowCount, DataGroupPart.State.INPROGRESS);
                             }
                         }

src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java

@@ -124,7 +124,6 @@ public static JSONObject toJsonTableMeta(TableMeta meta) {
         JSONObject tmeta = new JSONObject();
         tmeta.put("source", meta.getSource());
         tmeta.put("fileSize", meta.getFileSize());
-        tmeta.put("isFullyLoaded", meta.isLoaded());
         if (meta.getRelatedCols() != null) {
             tmeta.put("relatedCols", StringUtils.toString(meta.getRelatedCols(), ","));
         }

src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java

@@ -393,11 +393,11 @@ public static void sendLoadStatusEvents(Map<String,String> meta, File outf, int
         String source = meta.get("source") == null ? ServerContext.replaceWithPrefix(outf) : String.valueOf( meta.get("source") );
         String tblId = meta.get("tbl_id") == null ? source : String.valueOf( meta.get("tbl_id") );
 
-        FluxAction action = new FluxAction("table-space.loadTableStatus");
+        FluxAction action = new FluxAction("table_space.loadTableStatus");
         action.setValue(tblId, "tbl_id");
-        action.setValue(source, "source");
         action.setValue(crows, "totalRows");
         action.setValue(state.name(), "tableMeta", DataGroupPart.LOADING_STATUS);
+        action.setValue(source, "tableMeta", "source");
         ServerEventManager.fireAction(action);
     }
 

src/firefly/js/core/AppDataCntlr.js

@@ -1,7 +1,7 @@
 /*
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
-import {pick, isUndefined} from 'lodash';
+import {pickBy, isUndefined} from 'lodash';
 
 import {flux} from '../Firefly.js';
 import BrowserCache from '../util/BrowserCache.js';
@@ -322,7 +322,7 @@ function dispatchRemovePreference(name) {
  * @param hasXyPlots boolean. XY Plot data available.
  */
 function dispatchUpdateLayout({search, results, mode, views, hasTables, hasImages, hasXyPlots}) {
-    flux.process({type: UPDATE_LAYOUT, payload: pick({search, results, mode, views, hasTables, hasImages, hasXyPlots}, (v)=>(!isUndefined(v)))});
+    flux.process({type: UPDATE_LAYOUT, payload: pickBy({search, results, mode, views, hasTables, hasImages, hasXyPlots}, (v)=>(!isUndefined(v)))});
 }
 
 

src/firefly/js/core/History.js

@@ -3,8 +3,7 @@
  */
 
 import {flux} from '../Firefly.js';
-import set from 'lodash/object/set';
-import get from 'lodash/object/get';
+import {set, get} from 'lodash';
 
 var HISTORY_PATH = 'history';
 const MAX_HISTORY_LENGTH = 20;

src/firefly/js/core/ReduxFlux.js

@@ -17,7 +17,8 @@ import VisMouseCntlr from '../visualize/VisMouseCntlr.js';
 import TableStatsCntlr from '../visualize/TableStatsCntlr.js';
 import HistogramCntlr from '../visualize/HistogramCntlr.js';
 import XYPlotCntlr from '../visualize/XYPlotCntlr.js';
-import TablesCntlr from '../tables/TablesCntlr';
+import * as TablesCntlr from '../tables/TablesCntlr';
+import * as TablesUiCntlr from '../tables/TablesUiCntlr';
 import DrawLayer from '../visualize/DrawLayerCntlr.js';
 import DrawLayerFactory from '../visualize/draw/DrawLayerFactory.js';
 import DrawLayerCntlr, {makeDetachLayerActionCreator} from '../visualize/DrawLayerCntlr.js';
@@ -52,6 +53,7 @@ const reducers = {
     [HistogramCntlr.HISTOGRAM_DATA_KEY]: HistogramCntlr.reducer,
     [XYPlotCntlr.XYPLOT_DATA_KEY]: XYPlotCntlr.reducer,
     [TablesCntlr.TABLE_SPACE_PATH]: TablesCntlr.reducer,
+    [TablesUiCntlr.TABLE_UI_PATH]: TablesUiCntlr.reducer,
     [DrawLayer.DRAWING_LAYER_KEY]: DrawLayer.makeReducer(drawLayerFactory)
 };
 
@@ -66,10 +68,11 @@ actionCreators.set(ImagePlotCntlr.PLOT_IMAGE, ImagePlotCntlr.plotImageActionCrea
 actionCreators.set(ImagePlotCntlr.ZOOM_IMAGE, ImagePlotCntlr.zoomActionCreator);
 actionCreators.set(ImagePlotCntlr.COLOR_CHANGE, ImagePlotCntlr.colorChangeActionCreator);
 actionCreators.set(ImagePlotCntlr.STRETCH_CHANGE, ImagePlotCntlr.stretchChangeActionCreator);
-actionCreators.set(TablesCntlr.FETCH_TABLE, TablesCntlr.fetchTable);
-actionCreators.set(TablesCntlr.LOAD_TABLE, TablesCntlr.loadTable);
 actionCreators.set(DrawLayerCntlr.DETACH_LAYER_FROM_PLOT, makeDetachLayerActionCreator(drawLayerFactory));
 
+actionCreators.set(TablesCntlr.FETCH_TABLE, TablesCntlr.fetchTable);
+actionCreators.set(TablesCntlr.LOAD_TABLE, TablesCntlr.loadTable);
+actionCreators.set(TablesUiCntlr.TBL_UI_GOTO_PAGE, TablesUiCntlr.gotoPage);
 
 actionCreators.set(TableStatsCntlr.LOAD_TBL_STATS, TableStatsCntlr.loadTblStats);
 actionCreators.set(HistogramCntlr.LOAD_COL_DATA, HistogramCntlr.loadColData);

src/firefly/js/tables/SelectInfo.js

@@ -108,7 +108,7 @@ export class SelectInfo {
      * @returns {SelectInfo}
      */
     static newInstance({selectAll=false, exceptions=(new Set()), rowCount=0} = {}) {
-        return arguments[0] ? new SelectInfo(arguments[0]) : new SelectInfo({selectAll, exceptions, rowCount});
+        return new SelectInfo({selectAll, exceptions, rowCount});
     }
 
     /**
@@ -118,7 +118,7 @@ export class SelectInfo {
      * @returns {SelectInfo}
      */
     static find(tbl_id, root) {
-        var table = Table.find(tbl_id, root);
+        var table = Table.findTblById(tbl_id, root);
         return table && SelectInfo.newInstance(table.selectInfo);
     }
 

src/firefly/js/tables/Table.js

@@ -2,16 +2,17 @@
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
 
+import {get, slice} from 'lodash';
 import {flux} from '../Firefly.js';
-import TblCntlr from './TablesCntlr.js';
-import TblUtil from './TableUtil.js';
+import * as TblCntlr from './TablesCntlr.js';
+import * as TblUtil from './TableUtil.js';
 
 /**
  *
  */
 export class Table {
     constructor(dataModel) {
-        this.model = dataModel;
+        this.data = dataModel;
     }
 
     /**
@@ -20,11 +21,20 @@ export class Table {
      * @returns row of data(array of strings) or null if index is out of range.
      */
     getRow(idx) {
-        return TblUtil.find(this.model, 'tableData', 'data', idx);
+        return get(this.data, ['tableData.data', idx]);
     }
 
     getColumn(idx) {
-        return TblUtil.find(this.model, 'tableData', 'columns', idx);
+        return get(this.data, ['tableData.columns', idx]);
+    }
+
+    has(startIdx, endIdx) {
+        endIdx =  endIdx >0 ? Math.min( endIdx, this.data.totalRows) : startIdx;
+        if (startIdx >=0 && endIdx > startIdx) {
+            const data = get(this.data, 'tableData.data', []);
+            const aslice = data.slice(startIdx, endIdx).filter( (v) => v  );
+            return aslice.length === (endIdx-startIdx);
+        } else return false;
     }
 
     /**
@@ -36,14 +46,14 @@ export class Table {
     }
 
     /**
-     * return a Table with the data from the application's table-space.
+     * return a Table with the data from the application's table_space.
      * @param tbl_id unique table ID
-     * @param root the application state root.  If not given, flux.getState() will be used.
+     * @param root the data root.  If not given, flux.getState()[TblCntlr.TABLE_SPACE_PATH] will be used.
      * @returns {Table}
      */
-    static find(tbl_id, root) {
-        const state = root || flux.getState();
-        var table = TblUtil.find(state, TblCntlr.TABLE_SPACE_PATH, 'main', tbl_id);
+    static findTblById(tbl_id, root) {
+        const state = root || flux.getState()[TblCntlr.TABLE_SPACE_PATH];
+        var table = state[tbl_id];
         return table && Table.newInstance(table);
     }
 }

src/firefly/js/tables/TableRequest.js

@@ -2,7 +2,8 @@
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
 
-import {pick, identity} from 'lodash';
+import {pickBy} from 'lodash';
+import {uniqueTblId} from './TableUtil.js';
 
 
 export class TableRequest {
@@ -32,8 +33,8 @@ export class TableRequest {
      * @param copyFromReq
      * @returns {TableRequest}
      */
-    static newInstance({id, tbl_id, startIdx, pageSize, filters, sortInfo, inclCols, decimate, META_INFO, ...rest}, copyFromReq) {
-        var params = Object.assign(rest, pick({id, tbl_id, startIdx, pageSize, filters, sortInfo, inclCols, decimate, META_INFO}, identity));   // take only defined params
+    static newInstance({id, tbl_id=uniqueTblId(), startIdx, pageSize=100, filters, sortInfo, inclCols, decimate, META_INFO, ...rest}, copyFromReq) {
+        var params = Object.assign(rest, pickBy({id, tbl_id, startIdx, pageSize, filters, sortInfo, inclCols, decimate, META_INFO}));   // take only defined params
         if (copyFromReq) {
             params = Object.assign(copyFromReq, params);
         }