Skip to content

Commit

Permalink
[SPARK-13790] Speed up ColumnVector's getDecimal
Browse files Browse the repository at this point in the history
We should reuse an object similar to the other non-primitive type getters. For
a query that computes averages over decimal columns, this shows a 10% speedup
on overall query times.

TPCDS Snappy:                       Best/Avg Time(ms)    Rate(M/s)   Per Row(ns)
--------------------------------------------------------------------------------
q27-agg (master)                       10627 / 11057         10.8          92.3
q27-agg (this patch)                     9722 / 9832         11.8          84.4
  • Loading branch information
nongli committed Mar 10, 2016
1 parent e1772d3 commit 3b5ad12
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ final class Decimal extends Ordered[Decimal] with Serializable {
this
}

/**
* Just updates the underlying value to `v`, assuming precision and scale is unchanged.
*/
def setInternal(v: Long): Unit = {
this.longVal = v
}

/**
* Set this Decimal to the given unscaled Long, with a given precision and scale.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -536,9 +536,13 @@ private Array getByteArray(int rowId) {
*/
public final Decimal getDecimal(int rowId, int precision, int scale) {
if (precision <= Decimal.MAX_INT_DIGITS()) {
return Decimal.apply(getInt(rowId), precision, scale);
assert(resultDecimal != null);
resultDecimal.setInternal(getInt(rowId));
return resultDecimal;
} else if (precision <= Decimal.MAX_LONG_DIGITS()) {
return Decimal.apply(getLong(rowId), precision, scale);
assert (resultDecimal != null);
resultDecimal.setInternal(getLong(rowId));
return resultDecimal;
} else {
// TODO: best perf?
byte[] bytes = getBinary(rowId);
Expand Down Expand Up @@ -852,6 +856,11 @@ public final int appendStruct(boolean isNull) {
*/
protected final ColumnarBatch.Row resultStruct;

/**
* Reusable object for getDecimal()
*/
private Decimal resultDecimal;

/**
* The Dictionary for this column.
*
Expand Down Expand Up @@ -927,5 +936,12 @@ protected ColumnVector(int capacity, DataType type, MemoryMode memMode) {
this.resultArray = null;
this.resultStruct = null;
}

if (type instanceof DecimalType) {
DecimalType dt = (DecimalType)type;
if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) {
resultDecimal = Decimal.apply(0, dt.precision(), dt.scale());
}
}
}
}

0 comments on commit 3b5ad12

Please sign in to comment.