Skip to content

Commit

Permalink
Merge e0a1574 into afef232
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajantha-bhat
ajantha-bhat committed Jun 14, 2019
2 parents afef232 + e0a1574 commit e70f3b2
Show file tree
Hide file tree
Showing 18 changed files with 399 additions and 376 deletions.
2 changes: 1 addition & 1 deletion datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestAllOperationsOnMV.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
test("test block complex data types") {
sql("drop table IF EXISTS maintable")
sql("create table maintable(name string, c_code array<int>, price struct<b:int>,type map<string, string>) stored by 'carbondata'")
sql("insert into table maintable select 'abc',21,2000, 'ab\002type1'")
sql("insert into table maintable values('abc', array(21), named_struct('b', 2000), map('ab','type1'))")
sql("drop datamap if exists dm ")
intercept[UnsupportedOperationException] {
sql("create datamap dm using 'mv' as select c_code from maintable")
Expand Down
39 changes: 16 additions & 23 deletions .../src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,15 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
sql(
"create table test(person struct<detail:struct<id:int,name:string,height:double," +
"status:boolean,dob:date,dobt:timestamp>>) stored by 'carbondata'")
sql("insert into test values('1\002abc\0024.30\002true\0022017/08/09\0022017/08/09')")
sql("insert into test values(named_struct('detail', named_struct('id', 1, 'name', 'abc', 'height', 4.30, 'status', true, 'dob', '2017-08-09', 'dobt', '2017-08-09 00:00:00.0')))")
checkAnswer(sql("select * from test"),
Seq(Row(Row(Row(1, "abc", 4.3, true, java.sql.Date.valueOf("2017-08-09"),
Timestamp.valueOf("2017-08-09 00:00:00.0"))))))
sql("DROP TABLE IF EXISTS test")
sql(
"create table test(p1 array<int>,p2 array<string>,p3 array<double>,p4 array<boolean>,p5 " +
"array<date>,p6 array<timestamp>) stored by 'carbondata'")
sql(
"insert into test values('1\0012\0013','abc\001def\001mno','4.30\0014.60\0015.20','true\001true\001false'," +
"'2017/08/09\0012017/08/09\0012017/07/07','2017/08/09\0012017/08/09\0012017/07/07')")
sql("insert into test values(array(1,2,3), array('abc','def','mno'), array(4.30,4.60,5.20), array(true,true,false), array('2017-08-09','2017-08-09','2017-07-07'), array('2017-08-09 00:00:00.0','2017-08-09 00:00:00.0','2017-07-07 00:00:00.0'))")
checkAnswer(sql("select * from test"),
Seq(Row(mutable.WrappedArray.make(Array(1, 2, 3)),
mutable.WrappedArray.make(Array("abc", "def", "mno")),
Expand Down Expand Up @@ -142,7 +140,7 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"create table test(person struct<detail:struct<id:int,name:string,height:double," +
"status:boolean,dob:date,dobt:timestamp>>) stored by 'carbondata' tblproperties" +
"('dictionary_include'='person')")
sql("insert into test values('1\002abc\0024.30\002true\0022017/08/09\0022017/08/09')")
sql("insert into test values(named_struct('detail', named_struct('id', 1, 'name', 'abc', 'height', 4.30, 'status', true, 'dob', '2017-08-09', 'dobt', '2017-08-09 00:00:00.0')))")
checkAnswer(sql("select * from test"),
Seq(Row(Row(Row(1,
"abc", 4.3, true, java.sql.Date.valueOf("2017-08-09"),
Expand All @@ -152,9 +150,7 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"create table test(p1 array<int>,p2 array<string>,p3 array<double>,p4 array<boolean>,p5 " +
"array<date>,p6 array<timestamp>) stored by 'carbondata' tblproperties" +
"('dictionary_include'='p1,p2,p3,p4,p5,p6')")
sql(
"insert into test values('1\0012\0013','abc\001def\001mno','4.30\0014.60\0015.20','true\001true\001false'," +
"'2017/08/09\0012017/08/09\0012017/07/07','2017/08/09\0012017/08/09\0012017/07/07')")
sql("insert into test values(array(1,2,3), array('abc','def','mno'), array(4.30,4.60,5.20), array(true,true,false), array('2017-08-09','2017-08-09','2017-07-07'), array('2017-08-09 00:00:00.0','2017-08-09 00:00:00.0','2017-07-07 00:00:00.0'))")
checkAnswer(sql("select * from test"),
Seq(Row(mutable.WrappedArray.make(Array(1, 2, 3)),
mutable.WrappedArray.make(Array("abc", "def", "mno")),
Expand Down Expand Up @@ -206,9 +202,9 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"h:string,i:int>,j:int>) stored " +
"by " +
"'carbondata'")
sql("insert into complexcarbontable values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
sql("insert into complexcarbontable values(2,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
sql("insert into complexcarbontable values(3,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
sql("insert into complexcarbontable values(1, named_struct('b', 1, 'c', 'abc', 'd', 2, 'e', 'efg', 'f', named_struct('g', 3, 'h', 'mno', 'i', 4), 'j', 5))")
sql("insert into complexcarbontable values(2, named_struct('b', 1, 'c', 'abc', 'd', 2, 'e', 'efg', 'f', named_struct('g', 3, 'h', 'mno', 'i', 4), 'j', 5))")
sql("insert into complexcarbontable values(3, named_struct('b', 1, 'c', 'abc', 'd', 2, 'e', 'efg', 'f', named_struct('g', 3, 'h', 'mno', 'i', 4), 'j', 5))")
checkAnswer(sql("select a.b from complexcarbontable"), Seq(Row(1), Row(1), Row(1)))
checkAnswer(sql("select a.c from complexcarbontable"), Seq(Row("abc"), Row("abc"), Row("abc")))
checkAnswer(sql("select a.d from complexcarbontable"), Seq(Row(2), Row(2), Row(2)))
Expand Down Expand Up @@ -248,9 +244,9 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
test("test Complex_DataType-006") {
sql("DROP TABLE IF EXISTS test")
sql("create table test(id int,a struct<b:int,c:int>) stored by 'carbondata'")
sql("insert into test values(1,'2\0013')")
sql("insert into test values(3,'5\0013')")
sql("insert into test values(2,'4\0015')")
sql("insert into test values(1, named_struct('b', 2, 'c', 3))")
sql("insert into test values(3, named_struct('b', 5, 'c', 3))")
sql("insert into test values(2, named_struct('b', 4, 'c', 5))")
checkAnswer(sql("select a.b from test where id=3"), Seq(Row(5)))
checkAnswer(sql("select a.b from test where a.c!=3"), Seq(Row(4)))
checkAnswer(sql("select a.b from test where a.c=3"), Seq(Row(5), Row(2)))
Expand Down Expand Up @@ -294,28 +290,25 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"create table complexcarbontable(roll int, student struct<id:int,name:string," +
"marks:array<int>>) " +
"stored by 'carbondata'")
sql("insert into complexcarbontable values(1,'null\001abc\001null\002null\002null')")
sql("insert into complexcarbontable values(1, named_struct('id', 1, 'name', 'abc', 'marks', array(1,null,null)))")
checkAnswer(sql("select * from complexcarbontable"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
Seq(Row(1, Row(1, "abc", mutable.WrappedArray.make(Array(1, null, null))))))
}

//check create table with complex double and insert bigger value and check
test("test Complex_DataType-009") {
sql("Drop table if exists complexcarbontable")
sql(
"create table complexcarbontable(array1 array<struct<double1:double,double2:double," +
"double3:double>>) " +
"create table complexcarbontable(struct_dbl struct<double1:double,double2:double," +
"double3:double>) " +
"stored by 'carbondata'")
sql(
"insert into complexcarbontable values" +
"('14.35\002400000.35\0021.7976931348623157\00167890985.888\00265.5656\002200')," +
"('20.25\00250000.25\0024.945464565654656546546546324\00110000000\002300000\0023000')")
sql("insert into complexcarbontable values(named_struct('double1', 10000000, 'double2', 300000, 'double3', 3000))")
checkExistence(sql("select * from complexcarbontable"), true, "1.0E7,300000.0,3000.0")
sql("Drop table if exists complexcarbontable")
sql(
"create table complexcarbontable(struct_arr struct<array_db1:array<double>>) stored by " +
"'carbondata'")
sql("insert into complexcarbontable values('5555555.9559\00212345678991234567\0023444.999')")
sql("insert into complexcarbontable values(named_struct('array_db1', array(5555555.9559,12345678991234567,3444.999)))")
checkExistence(sql("select * from complexcarbontable"),
true,
"5555555.9559, 1.2345678991234568E16, 3444.999")
Expand Down
38 changes: 28 additions & 10 deletions ...t/scala/org/apache/carbondata/integration/spark/testsuite/binary/TestBinaryDataType.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties

import org.apache.commons.codec.binary.Hex
import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.commons.codec.binary.{Base64, Hex}
import org.apache.spark.SparkException
import org.apache.spark.sql.{AnalysisException, Row}
Expand Down Expand Up @@ -1458,7 +1459,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
""".stripMargin)

sql("INSERT INTO hive_table PARTITION(photo='binary') select 'a','b'");
sql("INSERT INTO hive_table PARTITION(photo=1) select 'a','b'");
sql("INSERT INTO hive_table PARTITION(photo='1') select 'a','b'");
checkAnswer(sql("select cast(photo as string) from hive_table"), Seq(Row("binary"), Row("1")));


Expand All @@ -1471,7 +1472,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
""".stripMargin)

sql("INSERT INTO hive_table2 PARTITION(photo='binary') select 'a','b'");
sql("INSERT INTO hive_table2 PARTITION(photo=1) select 'a','b'");
sql("INSERT INTO hive_table2 PARTITION(photo='1') select 'a','b'");
checkAnswer(sql("select cast(photo as string) from hive_table2"), Seq(Row("binary"), Row("1")));

sql(
Expand All @@ -1484,7 +1485,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
""".stripMargin)

sql("INSERT INTO parquet_table PARTITION(photo='binary') select 'a','b'");
sql("INSERT INTO parquet_table PARTITION(photo=1) select 'a','b'");
sql("INSERT INTO parquet_table PARTITION(photo='1') select 'a','b'");

sql("select cast(photo as string) from parquet_table").show()
//TODO： is it a bug in parquet?
Expand All @@ -1501,7 +1502,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {


sql("INSERT INTO carbon_partition_table PARTITION(photo='binary') select 'a','b'");
sql("INSERT INTO carbon_partition_table PARTITION(photo=1) select 'a','b'");
sql("INSERT INTO carbon_partition_table PARTITION(photo='1') select 'a','b'");
sql("select * from carbon_partition_table").show()
sql("select cast(photo as string) from carbon_partition_table").show()
checkAnswer(sql("select cast(photo as string) from carbon_partition_table"), Seq(Row("binary"), Row("1")))
Expand All @@ -1514,7 +1515,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
assert(e.getMessage.contains("Dynamic partition strict mode requires at least one static partition column"))

val eInt = intercept[Exception] {
sql("insert into hive_table select 'a','b',1");
sql("insert into hive_table select 'a','b','1'");
}

val e2 = intercept[SparkException] {
Expand All @@ -1524,7 +1525,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
assert(e2.getMessage.contains("Dynamic partition strict mode requires at least one static partition column"))

val eInt2 = intercept[Exception] {
sql("insert into hive_table2 select 'a','b',1");
sql("insert into hive_table2 select 'a','b','1'");
}

val e3 = intercept[SparkException] {
Expand All @@ -1534,11 +1535,11 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
assert(e3.getMessage.contains("Dynamic partition strict mode requires at least one static partition column"))

val eInt3 = intercept[Exception] {
sql("insert into parquet_table select 'a','b',1");
sql("insert into parquet_table select 'a','b','1'");
}

sql("insert into carbon_partition_table select 'a','b','binary'");
sql("insert into carbon_partition_table select 'a','b',1");
sql("insert into carbon_partition_table select 'a','b','1'");

checkAnswer(sql("select cast(photo as string) from carbon_partition_table"),
Seq(Row("binary"), Row("1"), Row("binary"), Row("1")))
Expand Down Expand Up @@ -1576,7 +1577,7 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
// Seq(Row(),Row(),Row()))

sql("insert into carbon_partition_table select 'a','b','binary'");
sql("insert into carbon_partition_table select 'a','b',1");
sql("insert into carbon_partition_table select 'a','b','1'");

checkAnswer(sql("select cast(photo as string) from carbon_partition_table"),
Seq(Row("binary"), Row("1"), Row("binary"), Row("1"), Row("binary"), Row("1")))
Expand Down Expand Up @@ -1715,6 +1716,23 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {

}

test("test binary insert with int value") {
sql("DROP TABLE IF EXISTS binaryTable")
sql(
s"""
| CREATE TABLE IF NOT EXISTS binaryTable (
| binaryField binary)
| STORED BY 'carbondata'
""".stripMargin)
val exception = intercept[AnalysisException] {
sql("insert into binaryTable select 1")
}
assert(exception.getMessage()
.contains(
"cannot resolve 'CAST(`1` AS BINARY)' due to data type mismatch: "))
sql("DROP TABLE binaryTable")
}

override def afterAll: Unit = {
sql("DROP TABLE IF EXISTS binaryTable")
sql("DROP TABLE IF EXISTS hiveTable")
Expand Down

0 comments on commit e70f3b2

Please sign in to comment.