Skip to content

DM-7271: table enhancements #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 24, 2016
Merged

DM-7271: table enhancements #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f707022
DM-7271: table enhancements
loitly Aug 19, 2016
d570eee
DM-7271: updates in response to reviews.
loitly Aug 23, 2016
7e3fa06
DM-7271: fix format not carried over when selected columns requested.
loitly Aug 24, 2016
fb2bea2
DM-7271: use double or long if type is not given for numeric data.
loitly Aug 24, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 60 additions & 17 deletions src/firefly/java/edu/caltech/ipac/astro/FITSTableReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,17 @@
import java.io.IOException;
import java.lang.reflect.Array;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* Convert an FITS file or FITS binary table(s) to list of DataGroup.
*/
public final class FITSTableReader
{
private static final Logger.LoggerImpl logger = Logger.getLogger();
private static Pattern TDISP = Pattern.compile("([ALIBOZFEGD])[NS]?(\\d+)?(\\.\\d+)?");


public static boolean debug = true;

Expand Down Expand Up @@ -275,21 +279,12 @@ public static DataGroup convertStarTableToDataGroup(StarTable table,
LinkedHashMap<ColumnInfo, Integer> colIdxMap = new LinkedHashMap<>();
List<String> colList = inclCols == null ? null : Arrays.asList(inclCols);

for (int col = 0; col < table.getColumnCount(); col++) {
ColumnInfo colInfo = table.getColumnInfo(col);
for (int colIdx = 0; colIdx < table.getColumnCount(); colIdx++) {
ColumnInfo colInfo = table.getColumnInfo(colIdx);
if ( colList == null || colList.contains(colInfo.getName() ) ) {
DataType dt = convertToDataType(colInfo, strategy);
if ( StringUtils.isEmpty(dt.getShortDesc()) ) {
// fill in column's description if not given
DescribedValue p = table.getParameterByName("TDOC" + (col+1)); // this is for LSST.. not sure it applies to others.
if (p != null) {
dt.setShortDesc(p.getValueAsString(200));
table.getParameters().remove(p);
}
}

DataType dt = convertToDataType(table, colIdx, strategy);
dataTypes.add(dt);
colIdxMap.put(colInfo, col);
colIdxMap.put(colInfo, colIdx);
}
}

Expand All @@ -300,10 +295,21 @@ public static DataGroup convertStarTableToDataGroup(StarTable table,
}

// setting DataGroup meta info
for(DataType dt : dataTypes) {
for(int colIdx = 0; colIdx < dataTypes.size(); colIdx++) {
DataType dt = dataTypes.get(colIdx);
if (!StringUtils.isEmpty(dt.getShortDesc())) {
dataGroup.addAttribute(DataSetParser.makeAttribKey(DataSetParser.DESC_TAG, dt.getKeyName()), dt.getShortDesc());
}
String format = getParam(table, "TDISP" + (colIdx + 1), 20);
format = format == null ? null : convertFormat(format);
if (Double.class.isAssignableFrom(dt.getDataType()) ||
Float.class.isAssignableFrom(dt.getDataType())) {
format = format == null && Double.class.isAssignableFrom(dt.getDataType()) ? "%.9g" : format;
dataGroup.addAttribute(DataSetParser.makeAttribKey(DataSetParser.FORMAT_TAG, dt.getKeyName()), "NONE");
}
if (!StringUtils.isEmpty(format)) {
dataGroup.addAttribute(DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName()), format);
}
}

List<String> hdList = inclHeaders == null ? null : Arrays.asList(inclHeaders);
Expand Down Expand Up @@ -360,18 +366,23 @@ private static void addRowToDataGroup(StarTable table, DataGroup dataGroup, Link
* (4)No change for other types
* Set unit.
*
* @param colInfo
*
* @param table
* @param colIdx
* @param strategy
* @return dataType
* @throws FitsException
*/
public static DataType convertToDataType(ColumnInfo colInfo, String strategy)
public static DataType convertToDataType(StarTable table, int colIdx, String strategy)
throws FitsException{

ColumnInfo colInfo = table.getColumnInfo(colIdx);
String colName = colInfo.getName();
String classType = DefaultValueInfo.formatClass(colInfo.getContentClass());
String unit = colInfo.getUnitString();
String nullString = null;
String desc = colInfo.getDescription();
desc = desc == null ? getParam(table, "TDOC" + (colIdx+1), 200) : desc; // this is for LSST.. not sure it applies to others.

DataType dataType = new DataType(colName, null);
Class java_class = null;
Expand Down Expand Up @@ -406,11 +417,43 @@ public static DataType convertToDataType(ColumnInfo colInfo, String strategy)
dataType.setDataType(java_class);
dataType.setUnits(unit);
dataType.setNullString(nullString);
dataType.setShortDesc(colInfo.getDescription());
dataType.setShortDesc(desc);

return dataType;
}

private static String getParam(StarTable table, String key, int maxWidth) {
DescribedValue p = table.getParameterByName(key);
return p == null ? null : p.getValueAsString(maxWidth);
}

/**
* converts FITS table keyword TDISPn into java format
* see http://archive.stsci.edu/fits/fits_standard/node69.html#SECTION001232060000000000000
* @param format
* @return
*/
private static String convertFormat(String format) {
Matcher m = TDISP.matcher(format);
if (m.find()) {
int count = m.groupCount();
String conv = String.valueOf(m.group(1));
String width = ""; // count > 1 ? m.group(2) : ""; ignores width for now.
String prec = count > 2 ? m.group(3) : "";
width = width == null ? "" : width;
prec = prec == null ? "" : prec;
if (conv.matches("D|E")) {
return "%" + width + prec + "e";
} else if (conv.equals("G")) {
return "%" + width + prec + "g";
} else if (conv.equals("F")) {
return "%" + width + prec + "f";
}
}
// not implemented or supported.. will print
return null;
}

/**
* Fill a dataGroup with a row of data from a table.
* Each dataObj contains data from all the columns at a repeat.
Expand Down
7 changes: 6 additions & 1 deletion src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,12 @@ protected static File convertToIpacTable(File tblFile, TableServerRequest reques
int tblIdx = request.getIntParam(TableServerRequest.TBL_INDEX, 0);
boolean isFixedLength = request.getBooleanParam(TableServerRequest.FIXED_LENGTH, true);
if (format == DataGroupReader.Format.IPACTABLE && isFixedLength) {
// file is already in ipac table format
TableDef tableDef = IpacTableUtil.getMetaInfo(tblFile);
if (tableDef.getCols().stream().anyMatch(c -> !c.isKnownType())) {
// if file missing types.. rewrite the file.
DataGroup dg = DataGroupReader.read(tblFile, true, false, true);
DataGroupWriter.write(tblFile, dg, 0);
}
return tblFile;
} else {
if ( format != DataGroupReader.Format.UNKNOWN) {
Expand Down
7 changes: 6 additions & 1 deletion src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,14 @@ private String getParam(String key, HashMap<String, String> params, HttpServletR

private File resolveDestDir(String dest, FileType fType) throws FileNotFoundException {
File destDir = ServerContext.getTempWorkDir();
/*
removed.. this writes temp file into the source directory. may be readonly. why was it needed before?
not sure of its history. leaving comment as a reminder in case it breaks something else.
if (!StringUtils.isEmpty(dest)) {
destDir = ServerContext.convertToFile(dest);
} else if (fType == FileType.FITS) {
} else
*/
if (fType == FileType.FITS) {
destDir = ServerContext.getVisCacheDir();
}
if (!destDir.exists()) {
Expand Down
28 changes: 17 additions & 11 deletions src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,7 @@
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
import edu.caltech.ipac.util.AppProperties;
import edu.caltech.ipac.util.CollectionUtil;
import edu.caltech.ipac.util.DataGroup;
import edu.caltech.ipac.util.DataGroupQuery;
import edu.caltech.ipac.util.DataObject;
import edu.caltech.ipac.util.DataObjectUtil;
import edu.caltech.ipac.util.DataType;
import edu.caltech.ipac.util.StringUtils;
import edu.caltech.ipac.util.*;
import edu.caltech.ipac.util.decimate.DecimateKey;

import javax.servlet.http.HttpServletRequest;
Expand Down Expand Up @@ -605,9 +598,22 @@ public static DataGroup doDecimation(DataGroup dg, DecimateInfo decimateInfo) {
Class xColClass = Double.class;
Class yColClass = Double.class;

ArrayList<DataGroup.Attribute> colMeta = new ArrayList<>();
try {
columns[0] = (!xValGetter.isExpression() ? dg.getDataDefintion(decimateInfo.getxColumnName()).copyWithNoColumnIdx(0) : new DataType("x", "x", xColClass, DataType.Importance.HIGH, "", false));
columns[1] = (!yValGetter.isExpression() ? dg.getDataDefintion(decimateInfo.getyColumnName()).copyWithNoColumnIdx(1) : new DataType("y", "y", yColClass, DataType.Importance.HIGH, "", false));
if (xValGetter.isExpression()) {
columns[0] = new DataType("x", "x", xColClass, DataType.Importance.HIGH, "", false);
} else {
columns[0] = dg.getDataDefintion(decimateInfo.getxColumnName()).copyWithNoColumnIdx(0);
colMeta.addAll(IpacTableUtil.getAllColMeta(dg.getAttributes().values(), decimateInfo.getxColumnName()));
}

if (yValGetter.isExpression()) {
columns[1] = new DataType("y", "y", yColClass, DataType.Importance.HIGH, "", false);
} else {
columns[1] = dg.getDataDefintion(decimateInfo.getyColumnName()).copyWithNoColumnIdx(1);
colMeta.addAll(IpacTableUtil.getAllColMeta(dg.getAttributes().values(), decimateInfo.getyColumnName()));
}

columns[2] = new DataType("rowidx", Integer.class); // need it to tie highlighted and selected to table
if (doDecimation) {
columns[3] = new DataType("weight", Integer.class);
Expand All @@ -621,7 +627,7 @@ public static DataGroup doDecimation(DataGroup dg, DecimateInfo decimateInfo) {


DataGroup retval = new DataGroup("decimated results", columns);

retval.setAttributes(colMeta);

// determine min/max values of x and y
boolean checkDeciLimits = false;
Expand Down
4 changes: 4 additions & 0 deletions src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ public static DataGroup read(File inf, boolean isFixedLength, boolean readAsStri
DataObject arow = new DataObject(data);
for (DataType dt : data.getDataDefinitions()) {
arow.setDataElement(dt, row.getDataElement(dt.getKeyName()));
if (dt.getFormatInfo().isDefault()) {
dt.getFormatInfo().setDataFormat(
headers.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr());
}
}
data.add(arow);
} else {
Expand Down
51 changes: 39 additions & 12 deletions src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,20 @@
*/
package edu.caltech.ipac.firefly.server.util.ipactable;

import edu.caltech.ipac.firefly.data.DecimateInfo;
import edu.caltech.ipac.firefly.data.Param;
import edu.caltech.ipac.firefly.data.SortInfo;
import edu.caltech.ipac.firefly.data.TableServerRequest;
import edu.caltech.ipac.firefly.data.table.TableMeta;
import edu.caltech.ipac.firefly.server.util.QueryUtil;
import edu.caltech.ipac.firefly.visualize.Band;
import edu.caltech.ipac.util.*;
import edu.jhu.util.StringUtil;
import edu.caltech.ipac.firefly.util.DataSetParser;
import edu.caltech.ipac.util.DataGroup;
import edu.caltech.ipac.util.DataType;
import edu.caltech.ipac.util.StringUtils;
import org.json.simple.JSONObject;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

import static edu.caltech.ipac.firefly.util.DataSetParser.*;

Expand Down Expand Up @@ -104,11 +104,38 @@ public static JSONObject toJsonTableRequest(TableServerRequest req) {
* @return
*/
public static JSONObject toJsonTableData(DataGroup data, TableDef tableDef) {
List<List<String>> tableData = new ArrayList<List<String>>();

// set display format if exists. this modifies DataType directly because it assumes it will no longer be used.
// if that is not the case, DataType will have to be cloned.
// also set flag to recalculate the max width of column's data
boolean formatChanged = false;
DataType[] columns = data.getDataDefinitions();
for (int colIdx = 0; colIdx < columns.length; colIdx++) {
DataType dt = columns[colIdx];
String fkey = DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName());
if (tableDef.contains(fkey)) {
dt.getFormatInfo().setDataFormat(tableDef.getAttribute(fkey).getValue());
dt.getFormatInfo().setWidth(Arrays.stream(new int[]{
String.valueOf(dt.getKeyName()).length(),
String.valueOf(dt.getDataUnit()).length(),
String.valueOf(dt.getTypeDesc()).length()}).max().getAsInt());
formatChanged = true;
}
}


List<List<String>> tableData = new ArrayList<>();
for (int i = 0; i < data.size(); i++) {
List<String> row = new ArrayList<String>();
for (Object o : data.get(i).getData()) {
row.add(String.valueOf(o));
List<String> row = new ArrayList<>();
String[] rowData = data.get(i).getFormatedData();
for (int colIdx = 0; colIdx < rowData.length; colIdx++) {
row.add(rowData[colIdx]);
if (formatChanged) {
DataType.FormatInfo fi = columns[colIdx].getFormatInfo();
int dlength = rowData[colIdx].length();
if (fi.getWidth() < dlength) fi.setWidth(dlength);
}

}
tableData.add(row);
}
Expand Down
5 changes: 5 additions & 0 deletions src/firefly/java/edu/caltech/ipac/firefly/util/DataSetParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,17 @@
* @version $Id: DataSetParser.java,v 1.18 2011/11/11 20:51:10 loi Exp $
*/
public class DataSetParser {
public static final String FMT_AUTO = "AUTO"; // guess format from data
public static final String FMT_NONE = "NONE"; // do not format data

public static final String LABEL_TAG = "col.@.Label";
public static final String VISI_TAG = "col.@.Visibility";
public static final String WIDTH_TAG = "col.@.Width";
public static final String PREF_WIDTH_TAG = "col.@.PrefWidth";
public static final String DESC_TAG = "col.@.ShortDescription";
public static final String UNIT_TAG = "col.@.Unit";
public static final String FORMAT_TAG = "col.@.Fmt"; // can be AUTO, NONE or a valid java format string. defaults to AUTO.
public static final String FORMAT_DISP_TAG = "col.@.FmtDisp";
public static final String SORTABLE_TAG = "col.@.Sortable";
public static final String ITEMS_TAG = "col.@.Items";
public static final String SORT_BY_TAG = "col.@.SortByCols";
Expand Down
19 changes: 2 additions & 17 deletions src/firefly/java/edu/caltech/ipac/util/DataGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,8 @@
*/
package edu.caltech.ipac.util;

import edu.jhu.util.StringUtil;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.*;

/**
* This class is the data class for any set of objects that we show on plots. <i>This class need more
Expand Down Expand Up @@ -140,12 +130,7 @@ public void shrinkToFitData(boolean force) {
int maxDataWidth = dt.getMaxDataWidth();
if (force || maxDataWidth == 0) {
for (DataObject row : this) {
int vlength = 0;
if (Number.class.isAssignableFrom(dt.getDataType())) {
vlength = dt.getFormatInfo().formatDataOnly(row.getDataElement(dt)).length();
} else {
vlength = String.valueOf(row.getDataElement(dt)).length();
}
int vlength = row.getDataWidth(dt);
maxDataWidth = Math.max(maxDataWidth, vlength);
}
}
Expand Down
17 changes: 17 additions & 0 deletions src/firefly/java/edu/caltech/ipac/util/DataObject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,23 @@ public String getFormatedData(DataType dt) {
return val;
}

/**
* returns the the length of the formatted string representation of the data.
* @param dt column info
* @return
*/
public int getDataWidth(DataType dt) {
int idx = dt.getColumnIdx();
String val;
if (_formattedData != null && _formattedData.length > idx && _formattedData[idx] != null) {
val = _formattedData[idx];
} else {
Object v = getDataElement(dt);
val = dt.getFormatInfo().formatDataOnly(v);
}
return val == null ? 0 : val.length();
}

public Object getDataElement(String name) {
checkSize();
DataType dtype = getDataType(name);
Expand Down
Loading