Skip to content

Commit

Permalink
ttools: implement Median Absolute Deviation in stats filter.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mbtaylor
mbtaylor committed Jun 14, 2013
1 parent 0a8a1c8 commit e1c45ae
Show file tree
Hide file tree
Showing 4 changed files with 246 additions and 24 deletions.
3 changes: 3 additions & 0 deletions ttools/src/docs/sun256.xml
View file
Expand Up @@ -8397,6 +8397,9 @@ eds. C. Gabriel et al., ASP Conf. Ser. 351, p. 666 (2006)
401s now generate a useful message about the
<code>star.basicauth.*</code> system properties if they
have not been set up.</li>
<li>Add Median Absolute Deviation calculation
(<code>MedAbsDev</code> and <code>ScMedAbsDev</code>)
options to <ref id="stats"><code>stats</code></ref> filter.</li>
</ul>
</p></dd>

Expand Down
206 changes: 185 additions & 21 deletions ttools/src/main/uk/ac/starlink/ttools/filter/QuantCalc.java
View file
Expand Up @@ -8,6 +8,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.TreeMap;
import uk.ac.starlink.table.Tables;

Expand Down Expand Up @@ -56,6 +57,24 @@ protected QuantCalc( Class clazz ) {
*/
public abstract Number getQuantile( double quant );

/**
* Returns the number of non-blank values accumulated by this calculator.
*
* @return value count
*/
public abstract long getValueCount();

/**
* Returns an iterator over all the non-blank values
* accumulated by this calculator.
* If {@link #ready} has been called, they will be in ascending order.
* The number of values it iterates over will be equal to
* the result of {@link #getValueCount}.
*
* @return value iterator
*/
public abstract Iterator<Number> getValueIterator();

/**
* Factory method to create a quantile accumulator for a given
* row count and value class.
Expand All @@ -73,8 +92,11 @@ public static QuantCalc createInstance( Class clazz, long nrow )
else if ( clazz == Short.class && ( nrow < 0 || nrow > ( 1 << 16 ) ) ) {
return new ShortSlotQuantCalc();
}
else if ( clazz == Integer.class || clazz == Long.class ) {
return new CountMapQuantCalc( clazz );
else if ( clazz == Integer.class ) {
return new CountMapQuantCalc( Integer.class );
}
else if ( clazz == Long.class ) {
return new CountMapQuantCalc( Long.class );
}
else if ( nrow >= 0 && nrow < Integer.MAX_VALUE ) {
return new FloatArrayQuantCalc( clazz, (int) nrow );
Expand All @@ -89,6 +111,26 @@ else if ( nrow >= Integer.MAX_VALUE ) {
}
}

/**
* Calculates the median absolute deviation of the statistics
* accumulated by a QuantCalc.
*
* @param qcalc calculator in ready state
* @return sum(abs(x_i - median))
*/
public static double calculateMedianAbsoluteDeviation( QuantCalc qcalc )
throws IOException {
double median = qcalc.getQuantile( 0.5 ).doubleValue();
QuantCalc madCalc =
QuantCalc.createInstance( Double.class, qcalc.getValueCount() );
for ( Iterator<Number> it = qcalc.getValueIterator(); it.hasNext(); ) {
double val = it.next().doubleValue();
madCalc.acceptDatum( Math.abs( val - median ) );
}
madCalc.ready();
return madCalc.getQuantile( 0.5 ).doubleValue();
}

/**
* QuantCalc implementation which uses an ArrayList of Number objects
* to keep track of the accumulated data. Not very efficient on
Expand All @@ -97,7 +139,7 @@ else if ( nrow >= Integer.MAX_VALUE ) {
static class ObjectListQuantCalc extends QuantCalc {

final Class clazz_;
final List list_;
final List<Number> list_;

/**
* Constructor.
Expand All @@ -107,27 +149,36 @@ static class ObjectListQuantCalc extends QuantCalc {
public ObjectListQuantCalc( Class clazz ) {
super( clazz );
clazz_ = clazz;
list_ = new ArrayList();
list_ = new ArrayList<Number>();
}

public void acceptDatum( Object obj ) {
if ( obj != null && obj.getClass().equals( clazz_ ) ) {
double dval = ((Number) obj).doubleValue();
Number num = ((Number) obj);
double dval = num.doubleValue();
if ( ! Double.isNaN( dval ) ) {
list_.add( obj );
list_.add( num );
}
}
}

public void ready() {
Collections.sort( list_ );
Collections.sort( (List) list_ );
}

public long getValueCount() {
return list_.size();
}

public Number getQuantile( double quant ) {
return list_.isEmpty()
? null
: (Number) list_.get( Math.min( (int) ( quant * list_.size() ),
list_.size() - 1 ) );
: list_.get( Math.min( (int) ( quant * list_.size() ),
list_.size() - 1 ) );
}

public Iterator<Number> getValueIterator() {
return list_.iterator();
}
}

Expand Down Expand Up @@ -159,6 +210,10 @@ public void ready() {
Arrays.sort( array_, 0, irow_ );
}

public long getValueCount() {
return irow_;
}

public Number getQuantile( double quant ) {
if ( irow_ == 0 ) {
return null;
Expand All @@ -184,6 +239,21 @@ else if ( clazz_ == Long.class ) {
return null;
}
}

public Iterator<Number> getValueIterator() {
return new Iterator<Number>() {
int i;
public boolean hasNext() {
return i < irow_;
}
public Number next() {
return new Float( array_[ i++ ] );
}
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}

/**
Expand Down Expand Up @@ -211,6 +281,10 @@ public void acceptDatum( Object obj ) {
public void ready() {
}

public long getValueCount() {
return count_;
}

public Number getQuantile( double quant ) {
long point = Math.min( (long) ( quant * count_ ), count_ - 1 );
long nval = 0;
Expand All @@ -222,6 +296,33 @@ public Number getQuantile( double quant ) {
}
return null;
}

public Iterator<Number> getValueIterator() {
return new Iterator<Number>() {
int is;
int ic;
Byte bval;
public boolean hasNext() {
while ( ic == 0 && is < slots_.length ) {
bval = new Byte( (byte) ( is - offset_ ) );
ic = slots_[ is++ ];
}
return ic > 0;
}
public Number next() {
if ( hasNext() ) {
ic--;
return bval;
}
else {
throw new NoSuchElementException();
}
}
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}

/**
Expand Down Expand Up @@ -249,6 +350,10 @@ public void acceptDatum( Object obj ) {
public void ready() {
}

public long getValueCount() {
return count_;
}

public Number getQuantile( double quant ) {
long point = Math.min( (long) ( quant * count_ ), count_ - 1 );
long nval = 0;
Expand All @@ -260,6 +365,33 @@ public Number getQuantile( double quant ) {
}
return null;
}

public Iterator<Number> getValueIterator() {
return new Iterator<Number>() {
int is;
int ic;
Short sval;
public boolean hasNext() {
while ( ic == 0 && is < slots_.length ) {
sval = new Short( (short) ( is - offset_ ) );
ic = slots_[ is++ ];
}
return ic > 0;
}
public Number next() {
if ( hasNext() ) {
ic--;
return sval;
}
else {
throw new NoSuchElementException();
}
}
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}

/**
Expand All @@ -268,46 +400,78 @@ public Number getQuantile( double quant ) {
*/
static class CountMapQuantCalc extends QuantCalc {
private final Class clazz_;
private Map countMap_;
private Map<Number,Integer> countMap_;
private long count_;
private static final Integer ONE = new Integer( 1 );

public CountMapQuantCalc( Class clazz ) {
public CountMapQuantCalc( Class<? extends Number> clazz ) {
super( clazz );
clazz_ = clazz;
countMap_ = new HashMap();
countMap_ = new HashMap<Number,Integer>();
}

public void acceptDatum( Object obj ) {
if ( obj != null && obj.getClass() == clazz_ &&
! Tables.isBlank( obj ) ) {
count_++;
Integer value = (Integer) countMap_.get( obj );
Number num = (Number) obj;
Integer value = countMap_.get( obj );
if ( value == null ) {
countMap_.put( obj, ONE );
countMap_.put( num, ONE );
}
else {
countMap_.put( obj, new Integer( value.intValue() + 1 ) );
countMap_.put( num, new Integer( value.intValue() + 1 ) );
}
}
}

public void ready() {
countMap_ = new TreeMap( countMap_ );
countMap_ = new TreeMap<Number,Integer>( countMap_ );
}

public long getValueCount() {
return count_;
}

public Number getQuantile( double quant ) {
long point = Math.min( (long) ( quant * count_ ), count_ - 1 );
long nval = 0;
for ( Iterator it = countMap_.entrySet().iterator();
it.hasNext(); ) {
Map.Entry entry = (Map.Entry) it.next();
nval += ((Integer) entry.getValue()).intValue();
for ( Map.Entry<Number,Integer> entry : countMap_.entrySet() ) {
nval += entry.getValue().intValue();
if ( nval > point ) {
return (Number) entry.getKey();
return entry.getKey();
}
}
return null;
}

public Iterator<Number> getValueIterator() {
return new Iterator<Number>() {
Iterator<Map.Entry<Number,Integer>> countIt =
countMap_.entrySet().iterator();
int ic;
Number num;
public boolean hasNext() {
while ( ic == 0 && countIt.hasNext() ) {
Map.Entry<Number,Integer> entry = countIt.next();
num = entry.getKey();
ic = entry.getValue().intValue();
}
return ic > 0;
}
public Number next() {
if ( hasNext() ) {
ic--;
return num;
}
else {
throw new NoSuchElementException();
}
}
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}
}

0 comments on commit e1c45ae

Please sign in to comment.