DM-7055: fix miscellaneous table issues

buildScript/depends.gincl

@@ -104,7 +104,9 @@ jar.destinationDir = file ("$rootDir/jars/build")
 
 task buildClient (dependsOn: loadConfig) {
 
-  outputs.upToDateWhen { false }
+  outputs.dir "${buildDir}/gwt/${project['app-name']}"
+  inputs.dir "${projectDir}/js"
+  inputs.dir "${fireflyPath}/src/firefly/js"
 
   doLast {
     def res = project.ext.NPM 'run', 'build'

src/firefly/java/edu/caltech/ipac/firefly/data/TableServerRequest.java

@@ -17,7 +17,7 @@ public class TableServerRequest extends ServerRequest implements Serializable, D
     public static final String DECIMATE_INFO = "decimate";
     public static final String TBL_FILE_PATH = "tblFilePath";
     public static final String TBL_ID = "tbl_id";
-    public static final String TITLE = "tbl_table";
+    public static final String TITLE = "title";
     public static final String FILTERS = "filters";
     public static final String SORT_INFO = "sortInfo";
     public static final String PAGE_SIZE = "pageSize";

src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java

@@ -199,8 +199,7 @@ public DataGroupPart getData(ServerRequest sr) throws DataAccessException {
                 try {
                     dgFile = postProcessData(dgFile, request);
                     page = IpacTableParser.getData(dgFile, request.getStartIndex(), request.getPageSize());
-                    page.getTableDef().ensureStatus();      // make sure there's a status line so
-                    page.getTableDef().setAttribute(TableServerRequest.TBL_FILE_PATH, ServerContext.replaceWithPrefix(dgFile));  // set table's meta tblFilePath to the file it came from.
+                    ensureTableMeta(page, request, dgFile);  // inspect/edit meta info needed by client.
                 } catch (Exception e) {
                     LOGGER.error(e, "Fail to parse ipac table file: " + dgFile);
                     throw e;
@@ -216,15 +215,14 @@ public DataGroupPart getData(ServerRequest sr) throws DataAccessException {
             LOGGER.error(e, "Error while processing request:" + StringUtils.truncate(sr, 512));
             throw new DataAccessException("Unexpected error", e);
         }
-//        finally {
-//            if (!doCache()) {
-// do not delete file even if it's not to be cached.  download still relies on it.
-//                if (dgFile != null) {
-//                    dgFile.delete();
-//                }
-//            }
-//        }
+    }
 
+    private void ensureTableMeta(DataGroupPart page, TableServerRequest request, File dgFile) {
+        page.getTableDef().ensureStatus();      // make sure there's a status line so
+        page.getTableDef().setAttribute(TableServerRequest.TBL_FILE_PATH, ServerContext.replaceWithPrefix(dgFile));  // set table's meta tblFilePath to the file it came from.
+        if (!StringUtils.isEmpty(request.getTblTitle())) {
+            page.getData().setTitle(request.getTblTitle());  // set the datagroup's title to the request title.
+        }
     }
 
     protected File postProcessData(File dgFile, TableServerRequest request) throws Exception {
@@ -648,3 +646,4 @@ protected static File convertToIpacTable(File tblFile, TableServerRequest reques
 
 }
 
+// some random comments

src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/IpacTableParser.java

@@ -87,8 +87,6 @@ public static DataGroupPart getData(File inf, int start, int rows) throws IOExce
         DataGroup dg = new DataGroup(null, tableDef.getCols());
         dg.setRowIdxOffset(start);
 
-        dg.setAttributes(tableDef.getAllAttributes());
-
         RandomAccessFile reader = new RandomAccessFile(inf, "r");
         long skip = ((long)start * (long)tableDef.getLineWidth()) + (long)tableDef.getRowStartOffset();
         int count = 0;
@@ -108,6 +106,13 @@ public static DataGroupPart getData(File inf, int start, int rows) throws IOExce
             reader.close();
         }
 
+        // sync attributes in datagroup with tabledef.
+        Map<String, DataGroup.Attribute> attribs = dg.getAttributes();
+        if (attribs.size() > 0) {
+            tableDef.addAttributes(attribs.values().toArray(new DataGroup.Attribute[attribs.size()]));
+        }
+        dg.setAttributes(tableDef.getAllAttributes());
+
         long totalRow = tableDef.getLineWidth() == 0 ? 0 :
                         (inf.length() - tableDef.getRowStartOffset())/tableDef.getLineWidth();
         return new DataGroupPart(tableDef, dg, start, (int) totalRow);

src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java

@@ -182,7 +182,7 @@ private static List<JSONObject> toJsonTableColumn(DataGroup dataGroup, TableDef
             }
             String sortable = getColAttr(tableDef, SORTABLE_TAG, cname);
             if (!StringUtils.isEmpty(sortable)) {
-                c.put("sortable", sortable);
+                c.put("sortable", Boolean.parseBoolean(sortable));
             }
             String units = getColAttr(tableDef, UNIT_TAG, cname);
             if (!StringUtils.isEmpty(units)) {

src/firefly/java/edu/caltech/ipac/util/DataGroup.java

@@ -183,6 +183,7 @@ public void remove(DataObject s) {
      */
     public void addAttribute(String key, String value) {
         _attributes.add(new Attribute(key, value));
+        _cachedAttributesMap = null;
     }
 
     /**

src/firefly/js/charts/HistogramCntlr.js

@@ -6,7 +6,7 @@ import {flux} from '../Firefly.js';
 import {get, has, omit} from 'lodash';
 
 import {updateSet, updateMerge} from '../util/WebUtil.js';
-import {doFetchTable, getTblById, isFullyLoaded, makeTblRequest} from '../tables/TableUtil.js';
+import {doFetchTable, getTblById, isFullyLoaded, makeTblRequest, cloneRequest} from '../tables/TableUtil.js';
 import {HISTOGRAM, getChartSpace} from './ChartUtil.js';
 import * as TablesCntlr from '../tables/TablesCntlr.js';
 import {DELETE} from './ChartsCntlr.js';
@@ -188,8 +188,7 @@ function fetchColData(dispatch, tblId, histogramParams, chartId) {
     const activeTableServerRequest = activeTableModel['request'];
     const tblSource = get(activeTableModel, 'tableMeta.tblFilePath');
 
-    const sreq = Object.assign({}, omit(activeTableServerRequest, ['tbl_id', 'META_INFO']),
-        {'startIdx' : 0, 'pageSize' : 1000000});
+    const sreq = cloneRequest(activeTableServerRequest, {'startIdx' : 0, 'pageSize' : 1000000});
 
     const req = makeTblRequest('HistogramProcessor');
     req.searchRequest = JSON.stringify(sreq);

src/firefly/js/charts/TableStatsCntlr.js

@@ -110,8 +110,7 @@ function fetchTblStats(dispatch, activeTableServerRequest) {
     const {tbl_id} = activeTableServerRequest;
 
     // searchRequest
-    const sreq = Object.assign({}, omit(activeTableServerRequest, ['tbl_id', 'META_INFO']),
-        {'startIdx': 0, 'pageSize': 1000000});
+    const sreq = TableUtil.cloneRequest(activeTableServerRequest, {'startIdx': 0, 'pageSize': 1000000});
 
     const req = TableUtil.makeTblRequest('StatisticsProcessor', `Statistics for ${tbl_id}`,
         { searchRequest: JSON.stringify(sreq) },

src/firefly/js/charts/XYPlotCntlr.js

@@ -6,7 +6,7 @@ import {flux} from '../Firefly.js';
 import {updateSet, updateMerge} from '../util/WebUtil.js';
 import {get, has, omit, omitBy, isUndefined, isString} from 'lodash';
 
-import {doFetchTable, getColumn, getTblById, isFullyLoaded} from '../tables/TableUtil.js';
+import {doFetchTable, getColumn, getTblById, isFullyLoaded, cloneRequest} from '../tables/TableUtil.js';
 import * as TablesCntlr from '../tables/TablesCntlr.js';
 import {DELETE} from './ChartsCntlr.js';
 import {serializeDecimateInfo} from '../tables/Decimate.js';
@@ -308,7 +308,7 @@ function fetchPlotData(dispatch, tblId, xyPlotParams, chartId) {
     if (!xyPlotParams) { xyPlotParams = getDefaultXYPlotParams(tblId); }
 
 
-    const req = Object.assign({}, omit(activeTableServerRequest, ['tbl_id', 'META_INFO']), {
+    const req = cloneRequest(activeTableServerRequest, {
             'startIdx' : 0,
             'pageSize' : 1000000,
             //'inclCols' : `${xyPlotParams.x.columnOrExpr},${xyPlotParams.y.columnOrExpr}`, // ignored if 'decimate' is present

src/firefly/js/charts/ui/ChartsTableViewPanel.jsx

@@ -335,7 +335,7 @@ class ChartsPanel extends React.Component {
                 filterInfoCls.setFilter(yCol, '> '+yMin);
                 filterInfoCls.addFilter(yCol, '< '+yMax);
                 const newRequest = Object.assign({}, tableModel.request, {filters: filterInfoCls.serialize()});
-                TablesCntlr.dispatchTableFetch(newRequest, 0);
+                TablesCntlr.dispatchTableFilter(newRequest, 0);
             }
         }
     }
@@ -344,7 +344,7 @@ class ChartsPanel extends React.Component {
         const request = get(this.props, 'tableModel.request');
         if (request && request.filters) {
             const newRequest = Object.assign({}, request, {filters: ''});
-            TablesCntlr.dispatchTableFetch(newRequest, 0);
+            TablesCntlr.dispatchTableFilter(newRequest, 0);
         }
     }
 
@@ -688,7 +688,7 @@ export class FilterEditorWrapper extends React.Component {
                         onChange={(obj) => {
                             if (!isUndefined(obj.filterInfo)) {
                                 const newRequest = Object.assign({}, tableModel.request, {filters: obj.filterInfo});
-                                TablesCntlr.dispatchTableFetch(newRequest, 0);
+                                TablesCntlr.dispatchTableFilter(newRequest, 0);
                             } else if (!isUndefined(obj.sortInfo)) {
                                 this.setState({sortInfo: obj.sortInfo});
                             }

src/firefly/js/core/ReduxFlux.js

@@ -133,6 +133,7 @@ actionCreators.set(DrawLayerCntlr.DETACH_LAYER_FROM_PLOT, makeDetachLayerActionC
 actionCreators.set(TablesCntlr.TABLE_SEARCH, TablesCntlr.tableSearch);
 actionCreators.set(TablesCntlr.TABLE_FETCH, TablesCntlr.tableFetch);
 actionCreators.set(TablesCntlr.TABLE_SORT, TablesCntlr.tableFetch);
+actionCreators.set(TablesCntlr.TABLE_FILTER, TablesCntlr.tableFetch);
 actionCreators.set(TablesCntlr.TABLE_HIGHLIGHT, TablesCntlr.highlightRow);
 
 actionCreators.set(TableStatsCntlr.LOAD_TBL_STATS, TableStatsCntlr.loadTblStats);

src/firefly/js/core/layout/FireflyLayoutManager.js

@@ -7,7 +7,7 @@ import {get, filter, omitBy, isNil} from 'lodash';
 
 import {LO_VIEW, LO_MODE, SHOW_DROPDOWN, SET_LAYOUT_MODE, getLayouInfo, dispatchUpdateLayoutInfo} from '../LayoutCntlr.js';
 import {clone} from '../../util/WebUtil.js';
-import {TBL_RESULTS_ADDED, TABLE_NEW, TABLE_REMOVE} from '../../tables/TablesCntlr.js';
+import {TBL_RESULTS_ADDED, TABLE_LOADED, TABLE_REMOVE} from '../../tables/TablesCntlr.js';
 import ImagePlotCntlr from '../../visualize/ImagePlotCntlr.js';
 import {isMetaDataTable, isCatalogTable} from '../../metaConvert/converterUtils.js';
 import {META_VIEWER_ID} from '../../visualize/ui/TriViewImageSection.jsx';
@@ -29,7 +29,7 @@ export function* layoutManager({title, views='tables | images | xyPlots'}) {
         const action = yield take([
             ImagePlotCntlr.PLOT_IMAGE_START, ImagePlotCntlr.PLOT_IMAGE,
             ImagePlotCntlr.DELETE_PLOT_VIEW, REPLACE_IMAGES,
-            TBL_RESULTS_ADDED, TABLE_REMOVE, TABLE_NEW,
+            TBL_RESULTS_ADDED, TABLE_REMOVE, TABLE_LOADED,
             SHOW_DROPDOWN, SET_LAYOUT_MODE
         ]);
 
@@ -55,7 +55,7 @@ export function* layoutManager({title, views='tables | images | xyPlots'}) {
                 ignore = handleLayoutChanges(action);
                 break;
 
-            case TABLE_NEW :
+            case TABLE_LOADED :
                 [showImages, images] = handleNewTable(action, images, showImages);
                 break;
 

src/firefly/js/drawingLayers/Catalog.js

@@ -15,7 +15,7 @@ import DrawLayerCntlr from '../visualize/DrawLayerCntlr.js';
 import {MouseState} from '../visualize/VisMouseSync.js';
 import DrawOp from '../visualize/draw/DrawOp.js';
 import {makeWorldPt} from '../visualize/Point.js';
-import {dispatchTableHighlight,dispatchTableFetch} from '../tables/TablesCntlr.js';
+import {dispatchTableHighlight,dispatchTableFilter} from '../tables/TablesCntlr.js';
 import {COLOR_HIGHLIGHTED_PT} from '../visualize/draw/DrawingDef.js';
 import {MetaConst} from '../data/MetaConst.js';
 import {SelectInfo} from '../tables/SelectInfo.js';
@@ -338,7 +338,7 @@ function doClearFilter(dl) {
     const tbl= getTblById(dl.drawLayerId);
     if (!tbl) return;
     const newRequest = Object.assign({}, tbl.request, {filters: ''});
-    dispatchTableFetch(newRequest);
+    dispatchTableFilter(newRequest);
 }
 
 
@@ -358,7 +358,7 @@ function doFilter(dl,p,sel) {
         filter= `IN (${idxs.toString()})`;
         filterInfoCls.addFilter(dl.tableMeta['decimate_key'], filter);
         newRequest = Object.assign({}, tbl.request, {filters: filterInfoCls.serialize()});
-        dispatchTableFetch(newRequest);
+        dispatchTableFilter(newRequest);
         console.log(newRequest);
         console.log(idxs);
     }
@@ -368,7 +368,7 @@ function doFilter(dl,p,sel) {
         // filterInfoCls.setFilter(filter);
         filterInfoCls.setFilter('ROWID', filter);
         newRequest = Object.assign({}, tbl.request, {filters: filterInfoCls.serialize()});
-        dispatchTableFetch(newRequest);
+        dispatchTableFilter(newRequest);
     }
 
 }

src/firefly/js/tables/TableConnector.js

@@ -35,7 +35,7 @@ export class TableConnector {
             // not implemented yet
         } else {
             request = Object.assign({}, request, {filters: filterIntoString});
-            TblCntlr.dispatchTableFetch(request);
+            TblCntlr.dispatchTableFilter(request);
         }
     }
 
@@ -59,7 +59,7 @@ export class TableConnector {
                         }, 'IN (') + ')';
                     filterInfoCls.addFilter('ROWID', value);
                     request = Object.assign({}, request, {filters: filterInfoCls.serialize()});
-                    TblCntlr.dispatchTableFetch(request);
+                    TblCntlr.dispatchTableFilter(request);
                 });
             }
         }

src/firefly/js/tables/TableUtil.js

@@ -2,7 +2,7 @@
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
 
-import {get, set, isEmpty, uniqueId, cloneDeep, omit, omitBy, isNil, isPlainObject, isArray} from 'lodash';
+import {get, set, unset, isEmpty, uniqueId, cloneDeep, omit, omitBy, isNil, isPlainObject, isArray} from 'lodash';
 import * as TblCntlr from './TablesCntlr.js';
 import {SortInfo, SORT_ASC, UNSORTED} from './SortInfo.js';
 import {flux} from '../Firefly.js';
@@ -132,6 +132,18 @@ export function makeVOCatalogRequest(title, params={}, options={}) {
     return omitBy(Object.assign(req, options, params, {id, tbl_id, META_INFO, UserTargetWorldPt}), isNil);
 }
 
+/**
+ * create a deep clone of the given request.  tbl_id is removed from the cloned request.
+ * @param {Object} request  the original request to clone
+ * @param {Object} params   additional parameters to add to the cloned request
+ * @returns {object}
+ */
+export function cloneRequest(request, params = {}) {
+    const req = cloneDeep(omit(request, 'tbl_id'));
+    unset(req, 'META_INFO.tbl_id');
+    return Object.assign(req, params);
+}
+
 /*---------------------------- creator functions >----------------------------*/
 
 
@@ -478,7 +490,8 @@ export function getTableSourceUrl(columns, request, filename) {
 }
 
 /**
- * returns a map of cname -> width.  The width is the number of characters needed to display
+ * returns an array of width indexed corresponding to the given columns.  
+ * The width is the number of characters needed to display
  * the header and the data as a table given columns and dataAry.
  * @param columns  array of column object
  * @param dataAry  array of array.
@@ -491,7 +504,7 @@ export function calcColumnWidths(columns, dataAry) {
         width = dataAry.reduce( (maxWidth, row) => {
             return Math.max(maxWidth, get(row, [idx, 'length'], 0));
         }, width);  // max width of data
-        pv[cname] = width;
+        pv[idx] = width;
         return pv;
     }, {ROWID: 8});
 }