From d91d93936ce1c12765dffdbf2fe773585dd39eae Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 17 Apr 2018 20:15:44 +0200 Subject: [PATCH 1/5] Added a test for checking from_json: json -> struct of map --- .../sql/catalyst/expressions/JsonExpressionsSuite.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 7812319756eae..9cfe7624297b2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -708,4 +708,12 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with } } } + + test("from_json support json to map[string, struct]") { + val input = """{"test":{"a":1}}""" + val schema = new StructType().add("test", MapType(StringType, IntegerType)) + val output = InternalRow(ArrayBasedMapData(Map("a" -> 1))) + + checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output) + } } From 428c6ba93202bf3235667794f9f8055c432733e1 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 17 Apr 2018 20:16:54 +0200 Subject: [PATCH 2/5] Added a test for comparison of ArrayBasedMapData --- .../sql/util/ArrayBasedMapDataSuite.scala | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala new file mode 100644 index 0000000000000..e19bb6fa78b58 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.util + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData} + +class DataTypeSuite extends SparkFunSuite { + test("compare 2 simple maps") { + val map1 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("a")), + valueArray = new GenericArrayData(Array[Int](1)) + ) + val map2 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("a")), + valueArray = new GenericArrayData(Array[Int](1)) + ) + + assert(map1 == map2) + } +} \ No newline at end of file From 58d1f7e41883b5e8924eca83bcf72deedadaf0ef Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 17 Apr 2018 21:01:00 +0200 Subject: [PATCH 3/5] Implemented the equals and hashCode methods of ArrayBasedMapData --- .../sql/catalyst/util/ArrayBasedMapData.scala | 12 ++++ .../expressions/JsonExpressionsSuite.scala | 4 +- .../sql/util/ArrayBasedMapDataSuite.scala | 61 +++++++++++++++++-- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala index 91b3139443696..3f204319cfb35 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapData.scala @@ -29,6 +29,18 @@ class ArrayBasedMapData(val keyArray: ArrayData, val valueArray: ArrayData) exte override def toString: String = { s"keys: $keyArray, values: $valueArray" } + + override def equals(that: Any): Boolean = that match { + case m: ArrayBasedMapData => this.keyArray == m.keyArray && this.valueArray == m.valueArray + case _ => false + } + + override def hashCode(): Int = { + val keyHashCode = if (keyArray == null) 0 else keyArray.hashCode() + val valueHashCode = if (valueArray == null) 0 else valueArray.hashCode() + + 41 * keyHashCode + valueHashCode + } } object ArrayBasedMapData { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 9cfe7624297b2..d3e9cb45e5bd5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -709,10 +709,10 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with } } - test("from_json support json to map[string, struct]") { + test("from_json support json to struct with map[string, struct]") { val input = """{"test":{"a":1}}""" val schema = new StructType().add("test", MapType(StringType, IntegerType)) - val output = InternalRow(ArrayBasedMapData(Map("a" -> 1))) + val output = InternalRow(ArrayBasedMapData(Map(UTF8String.fromString("a") -> 1))) checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala index e19bb6fa78b58..52245c3b3cf75 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala @@ -20,17 +20,66 @@ package org.apache.spark.sql.util import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData} -class DataTypeSuite extends SparkFunSuite { - test("compare 2 simple maps") { +class ArrayBasedMapDataSuite extends SparkFunSuite { + test("compare simple maps") { val map1 = new ArrayBasedMapData( keyArray = new GenericArrayData(Array[String]("a")), - valueArray = new GenericArrayData(Array[Int](1)) - ) + valueArray = new GenericArrayData(Array[Int](1))) val map2 = new ArrayBasedMapData( keyArray = new GenericArrayData(Array[String]("a")), - valueArray = new GenericArrayData(Array[Int](1)) - ) + valueArray = new GenericArrayData(Array[Int](1))) assert(map1 == map2) + assert(map1.hashCode() == map2.hashCode()) + } + + test("compare not equal simple maps - values different") { + val map1 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("a")), + valueArray = new GenericArrayData(Array[Int](1))) + val map2 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("a")), + valueArray = new GenericArrayData(Array[Int](2))) + + assert(map1 != map2) + assert(map1.hashCode() != map2.hashCode()) + } + + test("compare not equal simple maps - keys different") { + val map1 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("a")), + valueArray = new GenericArrayData(Array[Int](1))) + val map2 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("b")), + valueArray = new GenericArrayData(Array[Int](1))) + + assert(map1 != map2) + assert(map1.hashCode() != map2.hashCode()) + } + + test("compare maps of maps") { + val baseMapA = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("A")), + valueArray = new GenericArrayData(Array[Int](1))) + val baseMapB = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array[String]("B")), + valueArray = new GenericArrayData(Array[Int](1))) + + val map1 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array(baseMapA, baseMapB)), + valueArray = new GenericArrayData(Array[Int](1, 2))) + val map2 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array(baseMapA, baseMapB)), + valueArray = new GenericArrayData(Array[Int](1, 2))) + + assert(map1 == map2) + assert(map1.hashCode() == map2.hashCode()) + + val map3 = new ArrayBasedMapData( + keyArray = new GenericArrayData(Array(baseMapA, baseMapB)), + valueArray = new GenericArrayData(Array[Int](1, 0))) + + assert(map1 != map3) + assert(map1.hashCode() != map3.hashCode()) } } \ No newline at end of file From 4391b0a94e7ddbf53043e6723b7e6e63655f5759 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 17 Apr 2018 21:21:49 +0200 Subject: [PATCH 4/5] Improving test title --- .../spark/sql/catalyst/expressions/JsonExpressionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index d3e9cb45e5bd5..5708532ec5164 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -709,7 +709,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with } } - test("from_json support json to struct with map[string, struct]") { + test("from_json support json to struct with map[string, integer] field") { val input = """{"test":{"a":1}}""" val schema = new StructType().add("test", MapType(StringType, IntegerType)) val output = InternalRow(ArrayBasedMapData(Map(UTF8String.fromString("a") -> 1))) From 1458077cf6817701d74fcebd2e83ab6a62889fd8 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 17 Apr 2018 21:34:24 +0200 Subject: [PATCH 5/5] Making Scala style checker happy --- .../org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala index 52245c3b3cf75..f7ba3535d0ff8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrayBasedMapDataSuite.scala @@ -82,4 +82,4 @@ class ArrayBasedMapDataSuite extends SparkFunSuite { assert(map1 != map3) assert(map1.hashCode() != map3.hashCode()) } -} \ No newline at end of file +}