Skip to content

Commit

Permalink
[SPARK-21312][SQL] correct offsetInBytes in UnsafeRow.writeToStream
Browse files Browse the repository at this point in the history
## What changes were proposed in this pull request?

Corrects offsetInBytes calculation in UnsafeRow.writeToStream. Known failures include writes to some DataSources that have own SparkPlan implementations and cause EXCHANGE in writes.

## How was this patch tested?

Extended UnsafeRowSuite.writeToStream to include an UnsafeRow over byte array having non-zero offset.

Author: Sumedh Wale <swale@snappydata.io>

Closes #18535 from sumwale/SPARK-21312.
  • Loading branch information
Sumedh Wale authored and cloud-fan committed Jul 6, 2017
1 parent 75b168f commit 14a3bb3
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 1 deletion.
Expand Up @@ -550,7 +550,7 @@ public void copyFrom(UnsafeRow row) {
*/
public void writeToStream(OutputStream out, byte[] writeBuffer) throws IOException {
if (baseObject instanceof byte[]) {
int offsetInByteArray = (int) (Platform.BYTE_ARRAY_OFFSET - baseOffset);
int offsetInByteArray = (int) (baseOffset - Platform.BYTE_ARRAY_OFFSET);
out.write((byte[]) baseObject, offsetInByteArray, sizeInBytes);
} else {
int dataRemaining = sizeInBytes;
Expand Down
13 changes: 13 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
Expand Up @@ -101,9 +101,22 @@ class UnsafeRowSuite extends SparkFunSuite {
MemoryAllocator.UNSAFE.free(offheapRowPage)
}
}
val (bytesFromArrayBackedRowWithOffset, field0StringFromArrayBackedRowWithOffset) = {
val baos = new ByteArrayOutputStream()
val numBytes = arrayBackedUnsafeRow.getSizeInBytes
val bytesWithOffset = new Array[Byte](numBytes + 100)
System.arraycopy(arrayBackedUnsafeRow.getBaseObject.asInstanceOf[Array[Byte]], 0,
bytesWithOffset, 100, numBytes)
val arrayBackedRow = new UnsafeRow(arrayBackedUnsafeRow.numFields())
arrayBackedRow.pointTo(bytesWithOffset, Platform.BYTE_ARRAY_OFFSET + 100, numBytes)
arrayBackedRow.writeToStream(baos, null)
(baos.toByteArray, arrayBackedRow.getString(0))
}

assert(bytesFromArrayBackedRow === bytesFromOffheapRow)
assert(field0StringFromArrayBackedRow === field0StringFromOffheapRow)
assert(bytesFromArrayBackedRow === bytesFromArrayBackedRowWithOffset)
assert(field0StringFromArrayBackedRow === field0StringFromArrayBackedRowWithOffset)
}

test("calling getDouble() and getFloat() on null columns") {
Expand Down

0 comments on commit 14a3bb3

Please sign in to comment.