diff --git a/src/firefly/java/edu/caltech/ipac/astro/FITSTableReader.java b/src/firefly/java/edu/caltech/ipac/astro/FITSTableReader.java index 49be76dde7..17d5d043c4 100644 --- a/src/firefly/java/edu/caltech/ipac/astro/FITSTableReader.java +++ b/src/firefly/java/edu/caltech/ipac/astro/FITSTableReader.java @@ -16,6 +16,8 @@ import java.io.IOException; import java.lang.reflect.Array; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Convert an FITS file or FITS binary table(s) to list of DataGroup. @@ -23,6 +25,8 @@ public final class FITSTableReader { private static final Logger.LoggerImpl logger = Logger.getLogger(); + private static Pattern TDISP = Pattern.compile("([ALIBOZFEGD])[NS]?(\\d+)?(\\.\\d+)?"); + public static boolean debug = true; @@ -275,21 +279,12 @@ public static DataGroup convertStarTableToDataGroup(StarTable table, LinkedHashMap colIdxMap = new LinkedHashMap<>(); List colList = inclCols == null ? null : Arrays.asList(inclCols); - for (int col = 0; col < table.getColumnCount(); col++) { - ColumnInfo colInfo = table.getColumnInfo(col); + for (int colIdx = 0; colIdx < table.getColumnCount(); colIdx++) { + ColumnInfo colInfo = table.getColumnInfo(colIdx); if ( colList == null || colList.contains(colInfo.getName() ) ) { - DataType dt = convertToDataType(colInfo, strategy); - if ( StringUtils.isEmpty(dt.getShortDesc()) ) { - // fill in column's description if not given - DescribedValue p = table.getParameterByName("TDOC" + (col+1)); // this is for LSST.. not sure it applies to others. - if (p != null) { - dt.setShortDesc(p.getValueAsString(200)); - table.getParameters().remove(p); - } - } - + DataType dt = convertToDataType(table, colIdx, strategy); dataTypes.add(dt); - colIdxMap.put(colInfo, col); + colIdxMap.put(colInfo, colIdx); } } @@ -300,10 +295,21 @@ public static DataGroup convertStarTableToDataGroup(StarTable table, } // setting DataGroup meta info - for(DataType dt : dataTypes) { + for(int colIdx = 0; colIdx < dataTypes.size(); colIdx++) { + DataType dt = dataTypes.get(colIdx); if (!StringUtils.isEmpty(dt.getShortDesc())) { dataGroup.addAttribute(DataSetParser.makeAttribKey(DataSetParser.DESC_TAG, dt.getKeyName()), dt.getShortDesc()); } + String format = getParam(table, "TDISP" + (colIdx + 1), 20); + format = format == null ? null : convertFormat(format); + if (Double.class.isAssignableFrom(dt.getDataType()) || + Float.class.isAssignableFrom(dt.getDataType())) { + format = format == null && Double.class.isAssignableFrom(dt.getDataType()) ? "%.9g" : format; + dataGroup.addAttribute(DataSetParser.makeAttribKey(DataSetParser.FORMAT_TAG, dt.getKeyName()), "NONE"); + } + if (!StringUtils.isEmpty(format)) { + dataGroup.addAttribute(DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName()), format); + } } List hdList = inclHeaders == null ? null : Arrays.asList(inclHeaders); @@ -360,18 +366,23 @@ private static void addRowToDataGroup(StarTable table, DataGroup dataGroup, Link * (4)No change for other types * Set unit. * - * @param colInfo + * + * @param table + * @param colIdx * @param strategy * @return dataType * @throws FitsException */ - public static DataType convertToDataType(ColumnInfo colInfo, String strategy) + public static DataType convertToDataType(StarTable table, int colIdx, String strategy) throws FitsException{ + ColumnInfo colInfo = table.getColumnInfo(colIdx); String colName = colInfo.getName(); String classType = DefaultValueInfo.formatClass(colInfo.getContentClass()); String unit = colInfo.getUnitString(); String nullString = null; + String desc = colInfo.getDescription(); + desc = desc == null ? getParam(table, "TDOC" + (colIdx+1), 200) : desc; // this is for LSST.. not sure it applies to others. DataType dataType = new DataType(colName, null); Class java_class = null; @@ -406,11 +417,43 @@ public static DataType convertToDataType(ColumnInfo colInfo, String strategy) dataType.setDataType(java_class); dataType.setUnits(unit); dataType.setNullString(nullString); - dataType.setShortDesc(colInfo.getDescription()); + dataType.setShortDesc(desc); return dataType; } + private static String getParam(StarTable table, String key, int maxWidth) { + DescribedValue p = table.getParameterByName(key); + return p == null ? null : p.getValueAsString(maxWidth); + } + + /** + * converts FITS table keyword TDISPn into java format + * see http://archive.stsci.edu/fits/fits_standard/node69.html#SECTION001232060000000000000 + * @param format + * @return + */ + private static String convertFormat(String format) { + Matcher m = TDISP.matcher(format); + if (m.find()) { + int count = m.groupCount(); + String conv = String.valueOf(m.group(1)); + String width = ""; // count > 1 ? m.group(2) : ""; ignores width for now. + String prec = count > 2 ? m.group(3) : ""; + width = width == null ? "" : width; + prec = prec == null ? "" : prec; + if (conv.matches("D|E")) { + return "%" + width + prec + "e"; + } else if (conv.equals("G")) { + return "%" + width + prec + "g"; + } else if (conv.equals("F")) { + return "%" + width + prec + "f"; + } + } + // not implemented or supported.. will print + return null; + } + /** * Fill a dataGroup with a row of data from a table. * Each dataObj contains data from all the columns at a repeat. diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java b/src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java index 0f791b97a4..a4d3e2e29e 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java @@ -622,7 +622,12 @@ protected static File convertToIpacTable(File tblFile, TableServerRequest reques int tblIdx = request.getIntParam(TableServerRequest.TBL_INDEX, 0); boolean isFixedLength = request.getBooleanParam(TableServerRequest.FIXED_LENGTH, true); if (format == DataGroupReader.Format.IPACTABLE && isFixedLength) { - // file is already in ipac table format + TableDef tableDef = IpacTableUtil.getMetaInfo(tblFile); + if (tableDef.getCols().stream().anyMatch(c -> !c.isKnownType())) { + // if file missing types.. rewrite the file. + DataGroup dg = DataGroupReader.read(tblFile, true, false, true); + DataGroupWriter.write(tblFile, dg, 0); + } return tblFile; } else { if ( format != DataGroupReader.Format.UNKNOWN) { diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java index f766bd280e..025dfb7612 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java @@ -101,9 +101,14 @@ private String getParam(String key, HashMap params, HttpServletR private File resolveDestDir(String dest, FileType fType) throws FileNotFoundException { File destDir = ServerContext.getTempWorkDir(); +/* + removed.. this writes temp file into the source directory. may be readonly. why was it needed before? + not sure of its history. leaving comment as a reminder in case it breaks something else. if (!StringUtils.isEmpty(dest)) { destDir = ServerContext.convertToFile(dest); - } else if (fType == FileType.FITS) { + } else +*/ + if (fType == FileType.FITS) { destDir = ServerContext.getVisCacheDir(); } if (!destDir.exists()) { diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java index e9baa76cef..96b014d1ad 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java @@ -25,14 +25,7 @@ import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart; import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader; import edu.caltech.ipac.firefly.server.util.ipactable.TableDef; -import edu.caltech.ipac.util.AppProperties; -import edu.caltech.ipac.util.CollectionUtil; -import edu.caltech.ipac.util.DataGroup; -import edu.caltech.ipac.util.DataGroupQuery; -import edu.caltech.ipac.util.DataObject; -import edu.caltech.ipac.util.DataObjectUtil; -import edu.caltech.ipac.util.DataType; -import edu.caltech.ipac.util.StringUtils; +import edu.caltech.ipac.util.*; import edu.caltech.ipac.util.decimate.DecimateKey; import javax.servlet.http.HttpServletRequest; @@ -605,9 +598,22 @@ public static DataGroup doDecimation(DataGroup dg, DecimateInfo decimateInfo) { Class xColClass = Double.class; Class yColClass = Double.class; + ArrayList colMeta = new ArrayList<>(); try { - columns[0] = (!xValGetter.isExpression() ? dg.getDataDefintion(decimateInfo.getxColumnName()).copyWithNoColumnIdx(0) : new DataType("x", "x", xColClass, DataType.Importance.HIGH, "", false)); - columns[1] = (!yValGetter.isExpression() ? dg.getDataDefintion(decimateInfo.getyColumnName()).copyWithNoColumnIdx(1) : new DataType("y", "y", yColClass, DataType.Importance.HIGH, "", false)); + if (xValGetter.isExpression()) { + columns[0] = new DataType("x", "x", xColClass, DataType.Importance.HIGH, "", false); + } else { + columns[0] = dg.getDataDefintion(decimateInfo.getxColumnName()).copyWithNoColumnIdx(0); + colMeta.addAll(IpacTableUtil.getAllColMeta(dg.getAttributes().values(), decimateInfo.getxColumnName())); + } + + if (yValGetter.isExpression()) { + columns[1] = new DataType("y", "y", yColClass, DataType.Importance.HIGH, "", false); + } else { + columns[1] = dg.getDataDefintion(decimateInfo.getyColumnName()).copyWithNoColumnIdx(1); + colMeta.addAll(IpacTableUtil.getAllColMeta(dg.getAttributes().values(), decimateInfo.getyColumnName())); + } + columns[2] = new DataType("rowidx", Integer.class); // need it to tie highlighted and selected to table if (doDecimation) { columns[3] = new DataType("weight", Integer.class); @@ -621,7 +627,7 @@ public static DataGroup doDecimation(DataGroup dg, DecimateInfo decimateInfo) { DataGroup retval = new DataGroup("decimated results", columns); - + retval.setAttributes(colMeta); // determine min/max values of x and y boolean checkDeciLimits = false; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java index 1a8225715f..4fa6bfcd83 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java @@ -123,6 +123,10 @@ public static DataGroup read(File inf, boolean isFixedLength, boolean readAsStri DataObject arow = new DataObject(data); for (DataType dt : data.getDataDefinitions()) { arow.setDataElement(dt, row.getDataElement(dt.getKeyName())); + if (dt.getFormatInfo().isDefault()) { + dt.getFormatInfo().setDataFormat( + headers.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr()); + } } data.add(arow); } else { diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java index 3997f41176..2c98019688 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java @@ -3,20 +3,20 @@ */ package edu.caltech.ipac.firefly.server.util.ipactable; -import edu.caltech.ipac.firefly.data.DecimateInfo; import edu.caltech.ipac.firefly.data.Param; -import edu.caltech.ipac.firefly.data.SortInfo; import edu.caltech.ipac.firefly.data.TableServerRequest; -import edu.caltech.ipac.firefly.data.table.TableMeta; import edu.caltech.ipac.firefly.server.util.QueryUtil; -import edu.caltech.ipac.firefly.visualize.Band; -import edu.caltech.ipac.util.*; -import edu.jhu.util.StringUtil; +import edu.caltech.ipac.firefly.util.DataSetParser; +import edu.caltech.ipac.util.DataGroup; +import edu.caltech.ipac.util.DataType; +import edu.caltech.ipac.util.StringUtils; import org.json.simple.JSONObject; -import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import static edu.caltech.ipac.firefly.util.DataSetParser.*; @@ -104,11 +104,38 @@ public static JSONObject toJsonTableRequest(TableServerRequest req) { * @return */ public static JSONObject toJsonTableData(DataGroup data, TableDef tableDef) { - List> tableData = new ArrayList>(); + + // set display format if exists. this modifies DataType directly because it assumes it will no longer be used. + // if that is not the case, DataType will have to be cloned. + // also set flag to recalculate the max width of column's data + boolean formatChanged = false; + DataType[] columns = data.getDataDefinitions(); + for (int colIdx = 0; colIdx < columns.length; colIdx++) { + DataType dt = columns[colIdx]; + String fkey = DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName()); + if (tableDef.contains(fkey)) { + dt.getFormatInfo().setDataFormat(tableDef.getAttribute(fkey).getValue()); + dt.getFormatInfo().setWidth(Arrays.stream(new int[]{ + String.valueOf(dt.getKeyName()).length(), + String.valueOf(dt.getDataUnit()).length(), + String.valueOf(dt.getTypeDesc()).length()}).max().getAsInt()); + formatChanged = true; + } + } + + + List> tableData = new ArrayList<>(); for (int i = 0; i < data.size(); i++) { - List row = new ArrayList(); - for (Object o : data.get(i).getData()) { - row.add(String.valueOf(o)); + List row = new ArrayList<>(); + String[] rowData = data.get(i).getFormatedData(); + for (int colIdx = 0; colIdx < rowData.length; colIdx++) { + row.add(rowData[colIdx]); + if (formatChanged) { + DataType.FormatInfo fi = columns[colIdx].getFormatInfo(); + int dlength = rowData[colIdx].length(); + if (fi.getWidth() < dlength) fi.setWidth(dlength); + } + } tableData.add(row); } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/util/DataSetParser.java b/src/firefly/java/edu/caltech/ipac/firefly/util/DataSetParser.java index 4101467c09..9b0942623e 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/util/DataSetParser.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/util/DataSetParser.java @@ -26,12 +26,17 @@ * @version $Id: DataSetParser.java,v 1.18 2011/11/11 20:51:10 loi Exp $ */ public class DataSetParser { + public static final String FMT_AUTO = "AUTO"; // guess format from data + public static final String FMT_NONE = "NONE"; // do not format data + public static final String LABEL_TAG = "col.@.Label"; public static final String VISI_TAG = "col.@.Visibility"; public static final String WIDTH_TAG = "col.@.Width"; public static final String PREF_WIDTH_TAG = "col.@.PrefWidth"; public static final String DESC_TAG = "col.@.ShortDescription"; public static final String UNIT_TAG = "col.@.Unit"; + public static final String FORMAT_TAG = "col.@.Fmt"; // can be AUTO, NONE or a valid java format string. defaults to AUTO. + public static final String FORMAT_DISP_TAG = "col.@.FmtDisp"; public static final String SORTABLE_TAG = "col.@.Sortable"; public static final String ITEMS_TAG = "col.@.Items"; public static final String SORT_BY_TAG = "col.@.SortByCols"; diff --git a/src/firefly/java/edu/caltech/ipac/util/DataGroup.java b/src/firefly/java/edu/caltech/ipac/util/DataGroup.java index f91b9d13cf..a0eee023f6 100644 --- a/src/firefly/java/edu/caltech/ipac/util/DataGroup.java +++ b/src/firefly/java/edu/caltech/ipac/util/DataGroup.java @@ -3,18 +3,8 @@ */ package edu.caltech.ipac.util; -import edu.jhu.util.StringUtil; - import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * This class is the data class for any set of objects that we show on plots. This class need more @@ -140,12 +130,7 @@ public void shrinkToFitData(boolean force) { int maxDataWidth = dt.getMaxDataWidth(); if (force || maxDataWidth == 0) { for (DataObject row : this) { - int vlength = 0; - if (Number.class.isAssignableFrom(dt.getDataType())) { - vlength = dt.getFormatInfo().formatDataOnly(row.getDataElement(dt)).length(); - } else { - vlength = String.valueOf(row.getDataElement(dt)).length(); - } + int vlength = row.getDataWidth(dt); maxDataWidth = Math.max(maxDataWidth, vlength); } } diff --git a/src/firefly/java/edu/caltech/ipac/util/DataObject.java b/src/firefly/java/edu/caltech/ipac/util/DataObject.java index 55ff33781f..bd0b612881 100644 --- a/src/firefly/java/edu/caltech/ipac/util/DataObject.java +++ b/src/firefly/java/edu/caltech/ipac/util/DataObject.java @@ -121,6 +121,23 @@ public String getFormatedData(DataType dt) { return val; } + /** + * returns the the length of the formatted string representation of the data. + * @param dt column info + * @return + */ + public int getDataWidth(DataType dt) { + int idx = dt.getColumnIdx(); + String val; + if (_formattedData != null && _formattedData.length > idx && _formattedData[idx] != null) { + val = _formattedData[idx]; + } else { + Object v = getDataElement(dt); + val = dt.getFormatInfo().formatDataOnly(v); + } + return val == null ? 0 : val.length(); + } + public Object getDataElement(String name) { checkSize(); DataType dtype = getDataType(name); diff --git a/src/firefly/java/edu/caltech/ipac/util/DataType.java b/src/firefly/java/edu/caltech/ipac/util/DataType.java index 01ff351679..1a62c71349 100644 --- a/src/firefly/java/edu/caltech/ipac/util/DataType.java +++ b/src/firefly/java/edu/caltech/ipac/util/DataType.java @@ -232,6 +232,28 @@ else if (dt.equals(Boolean.class)) return _typeDesc; } + public static Class parseDataType(String type) { + switch (type) { + case DOUBLE: + case S_DOUBLE: + return Double.class; + case FLOAT: + case S_FLOAT: + return Float.class; + case LONG: + case S_LONG: + return Long.class; + case INTEGER: + case S_INTEGER: + return Integer.class; + case BOOL: + case S_BOOL: + return Boolean.class; + default: + return String.class; + } + } + public void setShortDesc(String shortDesc) { _shortDesc = shortDesc; } @@ -303,14 +325,11 @@ public enum Align {LEFT, RIGHT} private static final Align DEF_DATA_ALIGN = Align.LEFT; - private static final String DEF_FMT_STR = "%s"; - private static final int DEF_WIDTH = 30; private static final String NULL_STR = ""; private Align _headerAlign; private Align _dataAlign; private int _width; - private String _headerFormat; private String _dataFormat; private boolean _isDefault; @@ -380,20 +399,13 @@ public void setWidth(int width) { _width = width; } - public String getHeaderFormatStr() { - return _headerFormat == null ? DEF_FMT_STR : _headerFormat; - } - - public void setHeaderFormatStr(String headerFormatStr) { - _headerFormat = headerFormatStr; - } - public String getDataFormatStr() { - return _dataFormat == null ? DEF_FMT_STR : _dataFormat; + return _dataFormat; } public void setDataFormat(String dataFormatStr) { _dataFormat = dataFormatStr; + setIsDefault(false); } /** @@ -412,11 +424,7 @@ public String formatDataOnly(Object value) { */ public String formatDataOnly(Locale locale, Object value) { if (value == null) return NULL_STR; - String s = String.format(locale, getDataFormatStr(), value); - if (s.length() > getWidth()) { - s = s.substring(0, getWidth()); - } - return s; + return getDataFormatStr() == null ? String.valueOf(value) : String.format(locale, getDataFormatStr(), value); } /** @@ -444,7 +452,7 @@ public String formatData(Locale locale, Object value) { } public String formatData(Locale locale, Object value, String strForNull) { - String fmtStr = (getDataAlign() == Align.LEFT ? "%-" : "%") + getWidth() + "s"; + String fmtStr = (getDataAlign() == Align.LEFT ? "%-" : "%") + getWidth() + "." + getWidth()+ "s"; return String.format(locale, fmtStr, (value == null) ? strForNull : formatDataOnly(locale, value)); } @@ -455,8 +463,8 @@ public String formatData(Locale locale, Object value, String strForNull) { * @return a string */ public String formatHeader(String value) { - String v = value == null ? "" : String.format(getHeaderFormatStr(), value); - String fmtStr = (getHeaderAlign() == Align.LEFT ? "%-" : "%") + getWidth() + "s"; + String v = value == null ? "" : value; + String fmtStr = (getHeaderAlign() == Align.LEFT ? "%-" : "%") + getWidth() + "." + getWidth() + "s"; return String.format(fmtStr, v); } @@ -464,12 +472,6 @@ public Object clone() throws CloneNotSupportedException { return super.clone(); } - private static boolean isFloatingPoint(Class dt) { - return Double.class.isAssignableFrom(dt) || - Float.class.isAssignableFrom(dt); - } - - //========================================================================= // Convenience factory method to create common FormatInfo @@ -482,18 +484,11 @@ private static boolean isFloatingPoint(Class dt) { */ public static FormatInfo createDefaultFormat(Class dt) { - FormatInfo fi = null; dt = dt == null ? String.class : dt; - if (isFloatingPoint(dt)) { - fi = createFloatFormat(); - } else if (Date.class.isAssignableFrom(dt)) { + FormatInfo fi = new FormatInfo(); + fi.setDataAlign(Align.LEFT); + if (Date.class.isAssignableFrom(dt)) { fi = createDateFormat(); - } else if (Integer.class.isAssignableFrom(dt)) { - fi = new FormatInfo(); - fi.setDataFormat("%d"); - } else { - fi = new FormatInfo(); - fi.setDataAlign(Align.LEFT); } fi.setIsDefault(true); return fi; diff --git a/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java b/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java index c6afa3620b..01a4e3f407 100644 --- a/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java +++ b/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java @@ -4,17 +4,11 @@ package edu.caltech.ipac.util; import edu.caltech.ipac.astro.IpacTableReader; -import edu.caltech.ipac.firefly.data.ServerEvent; -import edu.caltech.ipac.firefly.server.ServerContext; import edu.caltech.ipac.firefly.server.events.FluxAction; import edu.caltech.ipac.firefly.server.events.ServerEventManager; import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart; import edu.caltech.ipac.firefly.server.util.ipactable.TableDef; -import edu.caltech.ipac.firefly.server.visualize.VisContext; import edu.caltech.ipac.firefly.util.DataSetParser; -import edu.caltech.ipac.firefly.util.event.Name; -import edu.jhu.util.StringUtil; -import org.json.simple.JSONObject; import java.io.BufferedReader; import java.io.File; @@ -22,11 +16,8 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Reader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; /** * Date: Jun 25, 2009 @@ -131,12 +122,36 @@ public static List createColumnDefs(String line) { return cols; } + /** + * return the meta info for the given cname column if it exists. + * @param metas a list of meta from the table + * @param cname the column name to search + * @param tag the column tag to search. see DataSetParser.java for a list of tags. + * @return + */ + public static String getColMeta(Collection metas, String cname, String tag) { + String mkey = DataSetParser.makeAttribKey(tag, cname); + Optional att = metas.stream().filter(m -> Objects.equals(m.getKey(), mkey)).findFirst(); + return att.isPresent() ? att.get().getValue() : null; + + } + + /** + * return the all meta info for the given cname column if it exists. + * @param metas a list of meta from the table + * @param cname the column name to search + * @return + */ + public static List getAllColMeta(Collection metas, String cname) { + return metas.stream().filter(m -> String.valueOf(m.getKey()).startsWith("col." + cname)).collect(Collectors.toList()); + } + public static void setDataType(List cols, String line) { if (line != null && line.startsWith("|")) { String[] types = parseHeadings(line.trim()); for (int i = 0; i < types.length; i++) { String typeDesc = types[i].trim(); - cols.get(i).setDataType(IpacTableReader.resolveClass(typeDesc)); + cols.get(i).setDataType(DataType.parseDataType(typeDesc)); } } } @@ -167,31 +182,18 @@ public static void guessFormatInfo(DataType dataType, String value) { DataType.FormatInfo fi = dataType.getFormatInfo(); fi.setDataFormat(formatStr); fi.setDataAlign(align); - fi.setIsDefault(false); } } public static void guessDataType(DataType type, String rval) { if (StringUtils.isEmpty(rval)) return; - try { - Integer.parseInt(rval); - type.setDataType(Integer.class); - return; - }catch (Exception e){} - try { Long.parseLong(rval); type.setDataType(Long.class); return; }catch (Exception e){} - try { - Float.parseFloat(rval); - type.setDataType(Float.class); - return; - }catch (Exception e){} - try { Double.parseDouble(rval); type.setDataType(Double.class); @@ -262,7 +264,12 @@ public static DataObject parseRow(DataGroup source, String line, boolean isFixed } offset = endoffset; if (type.getFormatInfo().isDefault()) { - IpacTableUtil.guessFormatInfo(type, rval); + DataGroup.Attribute format = source.getAttribute(DataSetParser.makeAttribKey(DataSetParser.FORMAT_TAG, type.getKeyName())); + if (format == null || Objects.equals(format.getValue(), DataSetParser.FMT_AUTO)) { + IpacTableUtil.guessFormatInfo(type, rval); + } else if (!Objects.equals(format.getValue(), DataSetParser.FMT_NONE)){ + type.getFormatInfo().setDataFormat(format.getValue()); + } // disable sorting if value is HTML, or unit is 'html' // this block should only be executed once, when formatInfo is not set. diff --git a/src/firefly/js/tables/TableUtil.js b/src/firefly/js/tables/TableUtil.js index fa6c2e097e..f5451cfd87 100644 --- a/src/firefly/js/tables/TableUtil.js +++ b/src/firefly/js/tables/TableUtil.js @@ -576,7 +576,7 @@ export function getTableSourceUrl(tbl_ui_id) { export function calcColumnWidths(columns, dataAry) { return columns.reduce( (pv, cv, idx) => { const cname = cv.name; - var width = Math.max(cname.length, get(cv, 'units.length', 0)); + var width = Math.max(cname.length, get(cv, 'units.length', 0), get(cv, 'type.length', 0)); width = dataAry.reduce( (maxWidth, row) => { return Math.max(maxWidth, get(row, [idx, 'length'], 0)); }, width); // max width of data diff --git a/src/firefly/test/edu/caltech/ipac/firefly/server/util/ipactable/test_data.json b/src/firefly/test/edu/caltech/ipac/firefly/server/util/ipactable/test_data.json index 4e6f165f49..555e20bf93 100644 --- a/src/firefly/test/edu/caltech/ipac/firefly/server/util/ipactable/test_data.json +++ b/src/firefly/test/edu/caltech/ipac/firefly/server/util/ipactable/test_data.json @@ -1 +1 @@ -{"request":{"RequestClass":"ServerRequest","META_INFO":{"test-meta":"test-meta-value"},"sortInfo":"ASC,ra,dec","startIdx":0,"decimate":"decimate=ra,dec,1234,0.5,,,,","pageSize":0,"id":"searchProcID","filters":"ra > 0;dec > 0"},"tableMeta":{"EQUINOX":"'J2000'","ORIGIN":"'IPAC Infrared Science Archive (IRSA), Caltech\/JPL'","DATETIME":"'2015-06-26 15:12:10'","DATABASE":"'2MASS All-Sky Point Source Catalog (PSC) (fp_psc)'","test-meta":"test-meta-value","SKYAREA":"'within 500.0 arcsec of ra=10.68479 dec=+41.26906 Eq J2000 '","source":"test data","DataTag":"'ADS\/IRSA.Gator#2015\/0626\/151210_19090'","StatusFile":"'\/workspace\/TMP_GFVzWe_19090\/Gator\/irsa\/19090\/log.19090.html'","fixlen":"T","RowsRetrieved":"2412","SQL":"'SELECT (39 column names follow in next row.)'"},"tableData":{"data":[["10.733387","41.214523","00h42m56.01s","41d12m52.28s","0.22","0.2","178","00425601+4112522","15.968","null","null","null","14.864","null","null","null","15.066","0.174","0.174","8.5","UUC","002","001","000","000004","2","0","n","1997-10-24","8","121.212","-21.629","0","null","null","null","null","0","null","236.330704","146.160473","null","null","null"],["10.73756","41.215744","00h42m57.01s","41d12m56.68s","0.25","0.21","81","00425701+4112566","16.313","0.165","0.165","9.7","14.736","null","null","null","14.256","null","null","null","CUU","200","100","000","050000","2","0","n","1997-10-24","8","121.215","-21.628","0","null","null","null","null","0","null","239.258783","143.324989","null","null","null"],["10.726409","41.210964","00h42m54.34s","41d12m39.47s","0.23","0.21","89","00425433+4112394","16.3","0.153","0.153","9.8","15.077","null","null","null","14.548","null","null","null","BUU","200","100","000","050000","2","0","n","1997-10-24","8","121.206","-21.632","0","null","null","null","null","0","null","237.560717","151.675477","null","null","null"],["10.739598","41.192741","00h42m57.50s","41d11m33.87s","0.2","0.17","84","00425750+4111338","16.232","0.123","0.123","10.5","15.445","0.142","0.143","9.9","15.161","0.169","0.169","7.8","BBC","222","111","000","060515","2","0","n","1997-10-24","8","121.216","-21.651","0","null","null","null","null","0","null","312.258965","151.60904","0.787","0.284","1.071"],["10.744471","41.197601","00h42m58.67s","41d11m51.36s","0.13","0.13","79","00425867+4111513","15.901","0.089","0.09","14.2","15.512","0.142","0.143","9.3","15.5","0.209","0.209","5.7","ABC","222","111","000","060506","2","0","n","1997-10-24","8","121.22","-21.646","0","null","null","null","null","0","null","303.785003","147.848202","0.389","0.012","0.401"],["10.684147","41.194","00h42m44.20s","41d11m38.40s","0.27","0.25","102","00424419+4111384","16.534","0.177","0.177","7.9","15.856","0.189","0.189","6.8","14.597","null","null","null","CCU","220","220","cc0","050500","2","0","n","1997-10-24","8","121.171","-21.648","0","null","null","null","null","0","null","270.221608","180.369332","0.678","null","null"],["10.683284","41.193066","00h42m43.99s","41d11m35.04s","0.29","0.25","87","00424398+4111350","16.398","0.155","0.155","9.0","15.957","0.21","0.21","6.2","14.526","null","null","null","BCU","220","220","cc0","060600","2","0","n","1997-10-24","8","121.17","-21.649","0","null","null","null","null","0","null","273.608782","180.854358","0.441","null","null"],["10.689488","41.18972","00h42m45.48s","41d11m22.99s","0.35","0.33","164","00424547+4111229","16.866","0.226","0.227","5.8","15.973","0.218","0.218","6.1","15.732","0.259","0.259","4.6","DDD","222","111","000","060605","2","0","n","1997-10-24","8","121.175","-21.652","0","null","null","null","null","0","null","285.907084","177.448577","0.893","0.241","1.134"],["10.692408","41.19532","00h42m46.18s","41d11m43.15s","0.28","0.25","7","00424617+4111431","16.698","0.195","0.196","6.8","15.765","0.18","0.18","7.4","14.947","null","null","null","CCU","220","110","000","060500","2","0","n","1997-10-24","8","121.178","-21.647","0","null","null","null","null","0","null","266.26399","175.554936","0.933","null","null"],["10.691879","41.191299","00h42m46.05s","41d11m28.68s","0.34","0.32","80","00424605+4111286","17.046","0.265","0.266","4.9","16.161","0.261","0.261","5.1","15.56","0.223","0.223","5.4","DDD","222","111","000","060606","2","0","n","1997-10-24","8","121.177","-21.651","0","null","null","null","null","0","null","280.596781","176.075518","0.885","0.601","1.486"],["10.699043","41.180546","00h42m47.77s","41d10m49.97s","0.08","0.07","90","00424777+4110499","13.648","0.025","0.027","113.0","13.367","0.023","0.025","67.2","13.372","0.035","0.036","40.5","AAA","222","111","000","666666","2","0","n","1997-10-24","8","121.182","-21.662","0","null","null","null","null","0","null","320.97889","173.089731","0.281","-0.005","0.276"],["10.699178","41.183144","00h42m47.80s","41d10m59.32s","0.36","0.34","180","00424780+4110593","17.027","0.241","0.242","5.0","15.803","0.174","0.174","7.1","15.078","null","null","null","DCU","220","110","c00","060300","2","0","n","1997-10-24","8","121.183","-21.659","0","null","null","null","null","0","null","311.74134","172.816453","1.224","null","null"],["10.702958","41.20282","00h42m48.71s","41d12m10.15s","0.09","0.08","90","00424870+4112101","15.364","0.056","0.057","23.3","14.701","0.067","0.068","19.7","14.693","0.102","0.102","12.0","AAA","222","111","000","060606","2","0","n","1997-10-24","8","121.186","-21.64","0","null","null","null","null","0","null","243.483459","168.339868","0.663","0.008","0.671"],["10.708817","41.194553","00h42m50.12s","41d11m40.39s","0.25","0.23","77","00425011+4111403","16.646","0.178","0.178","7.2","15.732","0.172","0.172","7.6","14.813","null","null","null","CCU","220","110","000","060500","2","0","n","1997-10-24","8","121.191","-21.648","0","null","null","null","null","0","null","276.000502","166.35987","0.914","null","null"],["10.710685","41.19548","00h42m50.56s","41d11m43.73s","0.22","0.21","119","00425056+4111437","16.027","0.096","0.097","12.6","15.654","0.155","0.156","8.2","15.255","0.173","0.173","7.2","ACC","222","111","000","060606","2","0","n","1997-10-24","8","121.192","-21.647","0","null","null","null","null","0","null","274.008477","165.167084","0.373","0.399","0.772"],["10.708745","41.198418","00h42m50.10s","41d11m54.30s","0.3","0.29","129","00425009+4111543","16.59","0.179","0.179","7.5","15.9","0.187","0.187","6.5","15.524","0.213","0.213","5.6","CCC","222","111","000","060506","2","0","n","1997-10-24","8","121.191","-21.644","0","null","null","null","null","0","null","262.450252","165.685719","0.69","0.376","1.066"]],"columns":[{"name":"ra","width":11,"units":"deg","type":"double"},{"name":"dec","width":10,"units":"deg","type":"double"},{"name":"clon","width":14,"type":"char"},{"name":"clat","width":14,"type":"char"},{"name":"err_maj","width":7,"units":"arcsec","type":"double"},{"name":"err_min","width":7,"units":"arcsec","type":"double"},{"name":"err_ang","width":7,"units":"deg","type":"int"},{"name":"designation","width":17,"type":"char"},{"name":"j_m","width":6,"units":"mag","type":"double"},{"name":"j_cmsig","width":7,"units":"mag","type":"double"},{"name":"j_msigcom","width":9,"units":"mag","type":"double"},{"name":"j_snr","width":10,"type":"double"},{"name":"h_m","width":6,"units":"mag","type":"double"},{"name":"h_cmsig","width":7,"units":"mag","type":"double"},{"name":"h_msigcom","width":9,"units":"mag","type":"double"},{"name":"h_snr","width":10,"type":"double"},{"name":"k_m","width":6,"units":"mag","type":"double"},{"name":"k_cmsig","width":7,"units":"mag","type":"double"},{"name":"k_msigcom","width":9,"units":"mag","type":"double"},{"name":"k_snr","width":10,"type":"double"},{"name":"ph_qual","width":7,"type":"char"},{"name":"rd_flg","width":6,"type":"char"},{"name":"bl_flg","width":6,"type":"char"},{"name":"cc_flg","width":6,"type":"char"},{"name":"ndet","width":6,"type":"char"},{"name":"gal_contam","width":10,"type":"int"},{"name":"mp_flg","width":6,"type":"int"},{"name":"hemis","width":5,"type":"c"},{"name":"xdate","width":10,"type":"char"},{"name":"scan","width":4,"type":"i"},{"name":"glon","width":7,"units":"deg","type":"double"},{"name":"glat","width":7,"units":"deg","type":"double"},{"name":"a","width":4,"type":"c"},{"name":"dist_opt","width":8,"units":"arcsec","type":"double"},{"name":"phi_opt","width":7,"units":"deg","type":"int"},{"name":"b_m_opt","width":7,"units":"mag","type":"double"},{"name":"vr_m_opt","width":8,"units":"mag","type":"double"},{"name":"nopt_mchs","width":9,"type":"int"},{"name":"ext_key","width":7,"type":"int"},{"name":"dist","width":12,"units":"arcsec","type":"double"},{"name":"angle","width":12,"units":"deg","type":"double"},{"name":"j_h","width":9,"type":"double"},{"name":"h_k","width":9,"type":"double"},{"name":"j_k","width":9,"type":"double"}]},"totalRows":16,"title":null,"type":"table"} \ No newline at end of file +{"request":{"RequestClass":"ServerRequest","META_INFO":{"test-meta":"test-meta-value"},"sortInfo":"ASC,ra,dec","startIdx":0,"decimate":"decimate=ra,dec,1234,0.5,,,,","pageSize":0,"id":"searchProcID","filters":"ra > 0;dec > 0"},"tableMeta":{"EQUINOX":"'J2000'","ORIGIN":"'IPAC Infrared Science Archive (IRSA), Caltech\/JPL'","DATETIME":"'2015-06-26 15:12:10'","DATABASE":"'2MASS All-Sky Point Source Catalog (PSC) (fp_psc)'","test-meta":"test-meta-value","SKYAREA":"'within 500.0 arcsec of ra=10.68479 dec=+41.26906 Eq J2000 '","source":"test data","DataTag":"'ADS\/IRSA.Gator#2015\/0626\/151210_19090'","StatusFile":"'\/workspace\/TMP_GFVzWe_19090\/Gator\/irsa\/19090\/log.19090.html'","fixlen":"T","RowsRetrieved":"2412","SQL":"'SELECT (39 column names follow in next row.)'"},"tableData":{"data":[["10.733387","41.214523","00h42m56.01s","41d12m52.28s","0.22","0.20","178","00425601+4112522","15.968","","","","14.864","","","","15.066","0.174","0.174","8.5","UUC","002","001","000","000004","2","0","n","1997-10-24","8","121.212","-21.629","0","","","","","0","","236.330704","146.160473","","",""],["10.737560","41.215744","00h42m57.01s","41d12m56.68s","0.25","0.21","81","00425701+4112566","16.313","0.165","0.165","9.7","14.736","","","","14.256","","","","CUU","200","100","000","050000","2","0","n","1997-10-24","8","121.215","-21.628","0","","","","","0","","239.258783","143.324989","","",""],["10.726409","41.210964","00h42m54.34s","41d12m39.47s","0.23","0.21","89","00425433+4112394","16.300","0.153","0.153","9.8","15.077","","","","14.548","","","","BUU","200","100","000","050000","2","0","n","1997-10-24","8","121.206","-21.632","0","","","","","0","","237.560717","151.675477","","",""],["10.739598","41.192741","00h42m57.50s","41d11m33.87s","0.20","0.17","84","00425750+4111338","16.232","0.123","0.123","10.5","15.445","0.142","0.143","9.9","15.161","0.169","0.169","7.8","BBC","222","111","000","060515","2","0","n","1997-10-24","8","121.216","-21.651","0","","","","","0","","312.258965","151.609040","0.787","0.284","1.071"],["10.744471","41.197601","00h42m58.67s","41d11m51.36s","0.13","0.13","79","00425867+4111513","15.901","0.089","0.09","14.2","15.512","0.142","0.143","9.3","15.500","0.209","0.209","5.7","ABC","222","111","000","060506","2","0","n","1997-10-24","8","121.220","-21.646","0","","","","","0","","303.785003","147.848202","0.389","0.012","0.401"],["10.684147","41.194000","00h42m44.20s","41d11m38.40s","0.27","0.25","102","00424419+4111384","16.534","0.177","0.177","7.9","15.856","0.189","0.189","6.8","14.597","","","","CCU","220","220","cc0","050500","2","0","n","1997-10-24","8","121.171","-21.648","0","","","","","0","","270.221608","180.369332","0.678","",""],["10.683284","41.193066","00h42m43.99s","41d11m35.04s","0.29","0.25","87","00424398+4111350","16.398","0.155","0.155","9.0","15.957","0.21","0.21","6.2","14.526","","","","BCU","220","220","cc0","060600","2","0","n","1997-10-24","8","121.170","-21.649","0","","","","","0","","273.608782","180.854358","0.441","",""],["10.689488","41.189720","00h42m45.48s","41d11m22.99s","0.35","0.33","164","00424547+4111229","16.866","0.226","0.227","5.8","15.973","0.218","0.218","6.1","15.732","0.259","0.259","4.6","DDD","222","111","000","060605","2","0","n","1997-10-24","8","121.175","-21.652","0","","","","","0","","285.907084","177.448577","0.893","0.241","1.134"],["10.692408","41.195320","00h42m46.18s","41d11m43.15s","0.28","0.25","7","00424617+4111431","16.698","0.195","0.196","6.8","15.765","0.18","0.18","7.4","14.947","","","","CCU","220","110","000","060500","2","0","n","1997-10-24","8","121.178","-21.647","0","","","","","0","","266.263990","175.554936","0.933","",""],["10.691879","41.191299","00h42m46.05s","41d11m28.68s","0.34","0.32","80","00424605+4111286","17.046","0.265","0.266","4.9","16.161","0.261","0.261","5.1","15.560","0.223","0.223","5.4","DDD","222","111","000","060606","2","0","n","1997-10-24","8","121.177","-21.651","0","","","","","0","","280.596781","176.075518","0.885","0.601","1.486"],["10.699043","41.180546","00h42m47.77s","41d10m49.97s","0.08","0.07","90","00424777+4110499","13.648","0.025","0.027","113.0","13.367","0.023","0.025","67.2","13.372","0.035","0.036","40.5","AAA","222","111","000","666666","2","0","n","1997-10-24","8","121.182","-21.662","0","","","","","0","","320.978890","173.089731","0.281","-0.005","0.276"],["10.699178","41.183144","00h42m47.80s","41d10m59.32s","0.36","0.34","180","00424780+4110593","17.027","0.241","0.242","5.0","15.803","0.174","0.174","7.1","15.078","","","","DCU","220","110","c00","060300","2","0","n","1997-10-24","8","121.183","-21.659","0","","","","","0","","311.741340","172.816453","1.224","",""],["10.702958","41.202820","00h42m48.71s","41d12m10.15s","0.09","0.08","90","00424870+4112101","15.364","0.056","0.057","23.3","14.701","0.067","0.068","19.7","14.693","0.102","0.102","12.0","AAA","222","111","000","060606","2","0","n","1997-10-24","8","121.186","-21.640","0","","","","","0","","243.483459","168.339868","0.663","0.008","0.671"],["10.708817","41.194553","00h42m50.12s","41d11m40.39s","0.25","0.23","77","00425011+4111403","16.646","0.178","0.178","7.2","15.732","0.172","0.172","7.6","14.813","","","","CCU","220","110","000","060500","2","0","n","1997-10-24","8","121.191","-21.648","0","","","","","0","","276.000502","166.359870","0.914","",""],["10.710685","41.195480","00h42m50.56s","41d11m43.73s","0.22","0.21","119","00425056+4111437","16.027","0.096","0.097","12.6","15.654","0.155","0.156","8.2","15.255","0.173","0.173","7.2","ACC","222","111","000","060606","2","0","n","1997-10-24","8","121.192","-21.647","0","","","","","0","","274.008477","165.167084","0.373","0.399","0.772"],["10.708745","41.198418","00h42m50.10s","41d11m54.30s","0.30","0.29","129","00425009+4111543","16.590","0.179","0.179","7.5","15.900","0.187","0.187","6.5","15.524","0.213","0.213","5.6","CCC","222","111","000","060506","2","0","n","1997-10-24","8","121.191","-21.644","0","","","","","0","","262.450252","165.685719","0.69","0.376","1.066"]],"columns":[{"name":"ra","width":11,"units":"deg","type":"double"},{"name":"dec","width":10,"units":"deg","type":"double"},{"name":"clon","width":14,"type":"char"},{"name":"clat","width":14,"type":"char"},{"name":"err_maj","width":7,"units":"arcsec","type":"double"},{"name":"err_min","width":7,"units":"arcsec","type":"double"},{"name":"err_ang","width":7,"units":"deg","type":"int"},{"name":"designation","width":17,"type":"char"},{"name":"j_m","width":6,"units":"mag","type":"double"},{"name":"j_cmsig","width":7,"units":"mag","type":"double"},{"name":"j_msigcom","width":9,"units":"mag","type":"double"},{"name":"j_snr","width":10,"type":"double"},{"name":"h_m","width":6,"units":"mag","type":"double"},{"name":"h_cmsig","width":7,"units":"mag","type":"double"},{"name":"h_msigcom","width":9,"units":"mag","type":"double"},{"name":"h_snr","width":10,"type":"double"},{"name":"k_m","width":6,"units":"mag","type":"double"},{"name":"k_cmsig","width":7,"units":"mag","type":"double"},{"name":"k_msigcom","width":9,"units":"mag","type":"double"},{"name":"k_snr","width":10,"type":"double"},{"name":"ph_qual","width":7,"type":"char"},{"name":"rd_flg","width":6,"type":"char"},{"name":"bl_flg","width":6,"type":"char"},{"name":"cc_flg","width":6,"type":"char"},{"name":"ndet","width":6,"type":"char"},{"name":"gal_contam","width":10,"type":"int"},{"name":"mp_flg","width":6,"type":"int"},{"name":"hemis","width":5,"type":"c"},{"name":"xdate","width":10,"type":"char"},{"name":"scan","width":4,"type":"i"},{"name":"glon","width":7,"units":"deg","type":"double"},{"name":"glat","width":7,"units":"deg","type":"double"},{"name":"a","width":4,"type":"c"},{"name":"dist_opt","width":8,"units":"arcsec","type":"double"},{"name":"phi_opt","width":7,"units":"deg","type":"int"},{"name":"b_m_opt","width":7,"units":"mag","type":"double"},{"name":"vr_m_opt","width":8,"units":"mag","type":"double"},{"name":"nopt_mchs","width":9,"type":"int"},{"name":"ext_key","width":7,"type":"int"},{"name":"dist","width":12,"units":"arcsec","type":"double"},{"name":"angle","width":12,"units":"deg","type":"double"},{"name":"j_h","width":9,"type":"double"},{"name":"h_k","width":9,"type":"double"},{"name":"j_k","width":9,"type":"double"}]},"totalRows":16,"title":null,"type":"table"} \ No newline at end of file