Skip to content

Commit

Permalink
[MINOR] Move map function test
Browse files Browse the repository at this point in the history 
This commit moves the map function tests for frames to functions/frame
instead of binary/frame, to colocate the frame testing.
  • Loading branch information
@Baunsgaard
Baunsgaard committed Nov 9, 2022
1 parent b0cdf5a commit 35b5ed0
Show file tree
Hide file tree
Showing 28 changed files with 358 additions and 149 deletions.
87 changes: 37 additions & 50 deletions src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
View file
Expand Up @@ -128,37 +128,30 @@ public FrameBlock(FrameBlock that) {
}

public FrameBlock(int ncols, ValueType vt) {
this();
_schema = UtilFunctions.nCopies(ncols, vt);
_colnames = null; //default not materialized
_colmeta = new ColumnMetadata[ncols];
for( int j=0; j<ncols; j++ )
_colmeta[j] = new ColumnMetadata(0);
this(UtilFunctions.nCopies(ncols, vt), null, null);
}

public FrameBlock(ValueType[] schema) {
this(schema, new String[0][]);
this(schema, null, null);
}

public FrameBlock(ValueType[] schema, String[] names) {
this(schema, names, new String[0][]);
this(schema, names, null);
}

public FrameBlock(ValueType[] schema, String[][] data) {
//default column names not materialized
this(schema, null, data);

}

public FrameBlock(ValueType[] schema, String[] names, String[][] data) {
_numRows = 0; //maintained on append
_schema = schema;
_colnames = names;
_colmeta = new ColumnMetadata[_schema.length];
for( int j=0; j<_schema.length; j++ )
_colmeta[j] = new ColumnMetadata(0);
for( int i=0; i<data.length; i++ )
appendRow(data[i]);
ensureAllocateMeta();
if(data != null)
for( int i=0; i<data.length; i++ )
appendRow(data[i]);
}

/**
Expand Down Expand Up @@ -298,8 +291,7 @@ public boolean isColumnMetadataDefault() {
}

public boolean isColumnMetadataDefault(int c) {
return _colmeta[c].getMvValue() == null
&& _colmeta[c].getNumDistinct() == 0;
return _colmeta[c].isDefault();
}

public void setColumnMetadata(ColumnMetadata[] colmeta) {
Expand Down Expand Up @@ -333,12 +325,7 @@ public void ensureAllocatedColumns(int numRows) {
_msize = -1;

// allocate column meta data if necessary
if( _colmeta == null || _schema.length != _colmeta.length ) {
_colmeta = new ColumnMetadata[_schema.length];
for( int j=0; j<_schema.length; j++ )
_colmeta[j] = new ColumnMetadata(0);
}

ensureAllocateMeta();
// early abort if already allocated
if( _coldata != null && _schema.length == _coldata.length ) {
//handle special case that to few rows allocated
Expand All @@ -360,6 +347,14 @@ public void ensureAllocatedColumns(int numRows) {
_numRows = numRows;
}

private void ensureAllocateMeta(){
if( _colmeta == null || _schema.length != _colmeta.length ) {
_colmeta = new ColumnMetadata[_schema.length];
for( int j=0; j<_schema.length; j++ )
_colmeta[j] = new ColumnMetadata();
}
}

/**
* Checks for matching column sizes in case of existing columns.
*
Expand Down Expand Up @@ -388,10 +383,6 @@ public static String createColName(int i) {
return "C" + i;
}

private String createNextColName(){
return _schema != null ? createColName(_schema.length) : createColName(0);
}

public boolean isColNamesDefault() {
boolean ret = (_colnames != null);
for( int j=0; j<getNumColumns() && ret; j++ )
Expand Down Expand Up @@ -447,7 +438,7 @@ public void reset(int nrow, boolean clearMeta) {
if( _colmeta != null ) {
for( int i=0; i<_colmeta.length; i++ )
if( !isColumnMetadataDefault(i) )
_colmeta[i] = new ColumnMetadata(0);
_colmeta[i] = new ColumnMetadata();
}
}
if(_coldata != null) {
Expand Down Expand Up @@ -572,19 +563,10 @@ public void appendColumn(double[] col) {
* @param vt The Value type
*/
private void appendColumnMetaData(ValueType vt){
appendColumnMetaData(vt, createNextColName());
}

/**
* Append the metadata associated with adding a column.
*
* @param vt The Value type
* @param colName The columnName
*/
private void appendColumnMetaData(ValueType vt, String colName){
if(_colnames != null)
_colnames = (String[]) ArrayUtils.add(getColumnNames(), createColName(_colnames.length+1));
_schema = (ValueType[]) ArrayUtils.add(_schema, vt);
_colnames = (String[]) ArrayUtils.add(getColumnNames(), colName);
// Since we append a column we reset the _msize
_colmeta = (ColumnMetadata[]) ArrayUtils.add(getColumnMetadata(), new ColumnMetadata());
_msize = -1;
}

Expand Down Expand Up @@ -1084,6 +1066,7 @@ public FrameBlock slice(int rl, int ru, int cl, int cu, boolean deep, CacheBlock
ret._coldata[j-cl].set(0, ru-rl, _coldata[j], rl);
}
}

return ret;
}

Expand Down Expand Up @@ -1174,7 +1157,7 @@ public FrameBlock append( FrameBlock that, FrameBlock ret, boolean cbind ) {
ret._colnames = (_colnames!=null) ? _colnames.clone() : null;
ret._colmeta = new ColumnMetadata[getNumColumns()];
for( int j=0; j<_schema.length; j++ )
ret._colmeta[j] = new ColumnMetadata(0);
ret._colmeta[j] = new ColumnMetadata();

//concatenate data (deep copy first, append second)
ret._coldata = new Array[getNumColumns()];
Expand Down Expand Up @@ -1436,24 +1419,23 @@ public FrameBlock dropInvalidType(FrameBlock schema) {
if(this.getNumColumns() != schema.getNumColumns())
throw new DMLException("mismatch in number of columns in frame and its schema "+this.getNumColumns()+" != "+schema.getNumColumns());

String[] schemaString = IteratorFactory.getStringRowIterator(this).next(); // extract the schema in String array
// extract the schema in String array
String[] schemaString = IteratorFactory.getStringRowIterator(schema).next();
for (int i = 0; i < this.getNumColumns(); i++) {
Array obj = this.getColumn(i);
String schemaCol = schemaString[i];
String type;
if(schemaCol.contains("FP")){
if(schemaCol.contains("FP"))
type = "FP";
} else if (schemaCol.contains("INT")){
else if(schemaCol.contains("INT"))
type = "INT";
} else if (schemaCol.contains("STRING")){
else if(schemaCol.contains("STRING"))
// In case of String columns, don't do any verification or replacements.
continue;
} else{
else
type = schemaCol;
}

for (int j = 0; j < this.getNumRows(); j++)
{

for (int j = 0; j < this.getNumRows(); j++){
if(obj.get(j) == null)
continue;
String dataValue = obj.get(j).toString().trim().replace("\"", "").toLowerCase() ;
Expand Down Expand Up @@ -1882,11 +1864,16 @@ private FrameBlock removeEmptyColumns(MatrixBlock select, boolean emptyReturn) {
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append("FrameBlock");
if(_colnames != null){
sb.append("\n");
sb.append(Arrays.toString(_colnames));
}
sb.append("\n");
sb.append(Arrays.toString(_schema));
sb.append("\n");
sb.append(Arrays.toString(_colmeta));
sb.append("\n");
sb.append(Arrays.toString(_coldata));

return sb.toString();
}
}
3 changes: 2 additions & 1 deletion src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java
View file
Expand Up @@ -22,6 +22,7 @@
import java.lang.ref.SoftReference;
import java.util.HashMap;

import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.io.Writable;
import org.apache.sysds.common.Types.ValueType;

Expand Down Expand Up @@ -86,7 +87,7 @@ public final int size(){

@Override
public String toString() {
return this.getClass().getSimpleName().toString() + ":" + _size;
return this.getClass().getSimpleName();
}

}
1 change: 1 addition & 0 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java
View file
Expand Up @@ -58,6 +58,7 @@ public static Array allocate(ValueType v, int nRow) {
return new StringArray(new String[nRow]);
case BOOLEAN:
return new BooleanArray(new boolean[nRow]);
case UINT8:
case INT32:
return new IntegerArray(new int[nRow]);
case INT64:
Expand Down
15 changes: 13 additions & 2 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java
View file
Expand Up @@ -120,8 +120,19 @@ public byte[] getAsByteArray(int nRow) {
return booleanBuffer.array();
}

@Override
public ValueType getValueType(){
@Override
public ValueType getValueType() {
return ValueType.BOOLEAN;
}

@Override
public String toString() {
StringBuilder sb = new StringBuilder(_data.length * 5 + 2);
sb.append(super.toString() + ":[");
for(int i = 0; i < _size - 1; i++)
sb.append((_data[i] ? 1 : 0) + ",");
sb.append(_data[_size - 1] ? 1 : 0);
sb.append("]");
return sb.toString();
}
}
46 changes: 36 additions & 10 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/ColumnMetadata.java
View file
Expand Up @@ -24,29 +24,38 @@
public class ColumnMetadata implements Serializable {
private static final long serialVersionUID = -90094082422100311L;

private long _ndistinct = 0;
private static final long DEFAULT_DISTINCT = -1;

private long _ndistinct = DEFAULT_DISTINCT;
private String _mvValue = null;

/**
* Default constructor
*/
public ColumnMetadata() {

}

public ColumnMetadata(long ndistinct) {
_ndistinct = ndistinct;
}
_ndistinct = ndistinct;
}

public ColumnMetadata(long ndistinct, String mvval) {
_ndistinct = ndistinct;
_mvValue = mvval;
}
_ndistinct = ndistinct <= 0 ? DEFAULT_DISTINCT: ndistinct;
_mvValue = mvval;
}

public ColumnMetadata(ColumnMetadata that) {
_ndistinct = that._ndistinct;
_mvValue = that._mvValue;
}
_ndistinct = that._ndistinct;
_mvValue = that._mvValue;
}

public long getNumDistinct() {
return _ndistinct;
}

public void setNumDistinct(long ndistinct) {
_ndistinct = ndistinct;
_ndistinct = ndistinct <= 0 ? DEFAULT_DISTINCT: ndistinct;
}

public String getMvValue() {
Expand All @@ -57,4 +66,21 @@ public void setMvValue(String mvVal) {
_mvValue = mvVal;
}

public boolean isDefault() {
return getMvValue() == null && getNumDistinct() == DEFAULT_DISTINCT;
}

@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(this.getClass().getSimpleName());
if(_ndistinct != DEFAULT_DISTINCT) {
sb.append(":");
sb.append(_ndistinct);
}
if(_mvValue != null)
sb.append("--" + _mvValue);

return sb.toString();
}
}
11 changes: 11 additions & 0 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java
View file
Expand Up @@ -124,4 +124,15 @@ public byte[] getAsByteArray(int nRow) {
public ValueType getValueType() {
return ValueType.FP64;
}

@Override
public String toString(){
StringBuilder sb = new StringBuilder(_data.length * 5 + 2);
sb.append(super.toString() + ":[");
for(int i = 0; i < _size-1; i++)
sb.append(_data[i] + ",");
sb.append(_data[_size-1]);
sb.append("]");
return sb.toString();
}
}
11 changes: 11 additions & 0 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java
View file
Expand Up @@ -123,4 +123,15 @@ public byte[] getAsByteArray(int nRow) {
public ValueType getValueType() {
return ValueType.FP32;
}

@Override
public String toString(){
StringBuilder sb = new StringBuilder(_data.length * 5 + 2);
sb.append(super.toString() + ":[");
for(int i = 0; i < _size-1; i++)
sb.append(_data[i] + ",");
sb.append(_data[_size-1]);
sb.append("]");
return sb.toString();
}
}
11 changes: 11 additions & 0 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java
View file
Expand Up @@ -123,4 +123,15 @@ public byte[] getAsByteArray(int nRow) {
public ValueType getValueType(){
return ValueType.INT32;
}

@Override
public String toString(){
StringBuilder sb = new StringBuilder(_data.length * 5 + 2);
sb.append(super.toString() + ":[");
for(int i = 0; i < _size-1; i++)
sb.append(_data[i] + ",");
sb.append(_data[_size-1]);
sb.append("]");
return sb.toString();
}
}
10 changes: 10 additions & 0 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java
View file
Expand Up @@ -124,4 +124,14 @@ public ValueType getValueType(){
return ValueType.INT64;
}

@Override
public String toString(){
StringBuilder sb = new StringBuilder(_data.length * 5 + 2);
sb.append(super.toString() + ":[");
for(int i = 0; i < _size-1; i++)
sb.append(_data[i] + ",");
sb.append(_data[_size-1]);
sb.append("]");
return sb.toString();
}
}
11 changes: 11 additions & 0 deletions src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java
View file
Expand Up @@ -115,4 +115,15 @@ public byte[] getAsByteArray(int nRow) {
public ValueType getValueType() {
return ValueType.STRING;
}

@Override
public String toString(){
StringBuilder sb = new StringBuilder(_data.length * 5 + 2);
sb.append(super.toString() + ":[");
for(int i = 0; i < _size-1; i++)
sb.append(_data[i] + ",");
sb.append(_data[_size-1]);
sb.append("]");
return sb.toString();
}
}

0 comments on commit 35b5ed0

Please sign in to comment.