From 0a7ebd7311c42707d2576cda9a7b28ce96d2973f Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 5 May 2015 14:35:59 -0700 Subject: [PATCH] Clarify reason why SqlSerializer2 supports this serializer --- .../main/scala/org/apache/spark/serializer/Serializer.scala | 3 ++- .../org/apache/spark/sql/execution/SparkSqlSerializer2.scala | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/serializer/Serializer.scala b/core/src/main/scala/org/apache/spark/serializer/Serializer.scala index dd4a3c090d647..4000f87d7ce12 100644 --- a/core/src/main/scala/org/apache/spark/serializer/Serializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/Serializer.scala @@ -86,7 +86,8 @@ abstract class Serializer { * serIn.open([obj2bytes] concatenate [obj1bytes]) should return (obj2, obj1) * }}} * - * In general, this property should hold for serializers that are stateless. + * In general, this property should hold for serializers that are stateless and that do not + * write special metadata at the beginning or end of the serialization stream. * * This API is private to Spark; this method should not be overridden in third-party subclasses * or called in user code and is subject to removal in future Spark releases. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlSerializer2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlSerializer2.scala index c841362a246ea..35ad987eb1a63 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlSerializer2.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlSerializer2.scala @@ -155,7 +155,10 @@ private[sql] class SparkSqlSerializer2(keySchema: Array[DataType], valueSchema: def newInstance(): SerializerInstance = new ShuffleSerializerInstance(keySchema, valueSchema) - override def supportsRelocationOfSerializedObjects: Boolean = true + override def supportsRelocationOfSerializedObjects: Boolean = { + // SparkSqlSerializer2 is stateless and writes no stream headers + true + } } private[sql] object SparkSqlSerializer2 {