Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-13098][SQL] remove GenericInternalRowWithSchema #10992

Closed
wants to merge 1 commit into from
Closed

[SPARK-13098][SQL] remove GenericInternalRowWithSchema #10992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,18 +233,6 @@ class GenericInternalRow(private[sql] val values: Array[Any]) extends BaseGeneri
override def copy(): GenericInternalRow = this
}

/**
* This is used for serialization of Python DataFrame
*/
class GenericInternalRowWithSchema(values: Array[Any], val schema: StructType)
extends GenericInternalRow(values) {

/** No-arg constructor for serialization. */
protected def this() = this(null, null)

def fieldIndex(name: String): Int = schema.fieldIndex(name)
}

class GenericMutableRow(values: Array[Any]) extends MutableRow with BaseGenericInternalRow {
/** No-arg constructor for serialization. */
protected def this() = this(null)
Expand Down
13 changes: 5 additions & 8 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/python.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ object EvaluatePython {
values(i) = toJava(row.get(i, struct.fields(i).dataType), struct.fields(i).dataType)
i += 1
}
new GenericInternalRowWithSchema(values, struct)
new GenericRowWithSchema(values, struct)

case (a: ArrayData, array: ArrayType) =>
val values = new java.util.ArrayList[Any](a.numElements())
Expand Down Expand Up @@ -199,10 +199,7 @@ object EvaluatePython {

case (c: Long, TimestampType) => c

case (c: String, StringType) => UTF8String.fromString(c)
case (c, StringType) =>
// If we get here, c is not a string. Call toString on it.
UTF8String.fromString(c.toString)
case (c, StringType) => UTF8String.fromString(c.toString)

case (c: String, BinaryType) => c.getBytes("utf-8")
case (c, BinaryType) if c.getClass.isArray && c.getClass.getComponentType.getName == "byte" => c
Expand Down Expand Up @@ -263,11 +260,11 @@ object EvaluatePython {
}

/**
* Pickler for InternalRow
* Pickler for external row.
*/
private class RowPickler extends IObjectPickler {

private val cls = classOf[GenericInternalRowWithSchema]
private val cls = classOf[GenericRowWithSchema]

// register this to Pickler and Unpickler
def register(): Unit = {
Expand All @@ -282,7 +279,7 @@ object EvaluatePython {
} else {
// it will be memorized by Pickler to save some bytes
pickler.save(this)
val row = obj.asInstanceOf[GenericInternalRowWithSchema]
val row = obj.asInstanceOf[GenericRowWithSchema]
// schema should always be same object for memoization
pickler.save(row.schema)
out.write(Opcodes.TUPLE1)
Expand Down