Skip to content

Commit

Permalink
fill zero when encoding binary
Browse files Browse the repository at this point in the history
  • Loading branch information
@marsishandsome
marsishandsome committed Jul 2, 2019
1 parent 04d4fa4 commit f8de3db
Show file tree
Hide file tree
Showing 5 changed files with 189 additions and 136 deletions.
212 changes: 87 additions & 125 deletions core/src/test/scala/com/pingcap/tispark/convert/ToBytesSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")

private def createTable(): Unit =
jdbcUpdate(
s"create table $dbtable(i INT, c2 VARBINARY(255), c3 TINYBLOB, c4 BLOB, c5 MEDIUMBLOB, c6 LONGBLOB)"
s"create table $dbtable(i INT, c1 BINARY(5), c2 VARBINARY(255), c3 TINYBLOB, c4 BLOB, c5 MEDIUMBLOB, c6 LONGBLOB)"
)

test("Test Convert from java.lang.Boolean to BYTES") {
Expand All @@ -29,14 +29,15 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val b: java.lang.Boolean = false

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)
val row2 = Row(2, a, b, a, b, a, b)
val row3 = Row(3, a, a, b, a, b, a)
val row4 = Row(4, b, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", BooleanType),
StructField("c2", BooleanType),
StructField("c3", BooleanType),
StructField("c4", BooleanType),
Expand All @@ -48,15 +49,19 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val readA: java.lang.Long = 49L
val readB: java.lang.Long = 48L

val readRow1 = Row(1, null, null, null, null, null)
val readRow2 = Row(2, readB, readA, readB, readA, readB)
val readRow3 = Row(3, readA, readB, readA, readB, readA)
val readRow4 = Row(4, readA, readA, readA, readA, readA)
val readRow5 = Row(5, readB, readB, readB, readB, readB)
val binaryReadA: Array[Byte] = Array(49.toByte, 0.toByte, 0.toByte, 0.toByte, 0.toByte)
val binaryReadB: Array[Byte] = Array(48.toByte, 0.toByte, 0.toByte, 0.toByte, 0.toByte)

val readRow1 = Row(1, null, null, null, null, null, null)
val readRow2 = Row(2, binaryReadA, readB, readA, readB, readA, readB)
val readRow3 = Row(3, binaryReadA, readA, readB, readA, readB, readA)
val readRow4 = Row(4, binaryReadB, readA, readA, readA, readA, readA)
val readRow5 = Row(5, binaryReadB, readB, readB, readB, readB, readB)

val readSchema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", LongType),
StructField("c2", LongType),
StructField("c3", LongType),
StructField("c4", LongType),
Expand Down Expand Up @@ -88,14 +93,15 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val b: java.lang.Byte = java.lang.Byte.valueOf("-22")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)
val row2 = Row(2, a, b, a, b, a, b)
val row3 = Row(3, a, a, b, a, b, a)
val row4 = Row(4, b, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", ByteType),
StructField("c2", ByteType),
StructField("c3", ByteType),
StructField("c4", ByteType),
Expand All @@ -107,15 +113,19 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val readA: Array[Byte] = Array(49.toByte, 49.toByte)
val readB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte)

val readRow1 = Row(1, null, null, null, null, null)
val readRow2 = Row(2, readB, readA, readB, readA, readB)
val readRow3 = Row(3, readA, readB, readA, readB, readA)
val readRow4 = Row(4, readA, readA, readA, readA, readA)
val readRow5 = Row(5, readB, readB, readB, readB, readB)
val binaryReadA: Array[Byte] = Array(49.toByte, 49.toByte, 0.toByte, 0.toByte, 0.toByte)
val binaryReadB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte, 0.toByte, 0.toByte)

val readRow1 = Row(1, null, null, null, null, null, null)
val readRow2 = Row(2, binaryReadA, readB, readA, readB, readA, readB)
val readRow3 = Row(3, binaryReadA, readA, readB, readA, readB, readA)
val readRow4 = Row(4, binaryReadB, readA, readA, readA, readA, readA)
val readRow5 = Row(5, binaryReadB, readB, readB, readB, readB, readB)

val readSchema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", BinaryType),
StructField("c2", BinaryType),
StructField("c3", BinaryType),
StructField("c4", BinaryType),
Expand Down Expand Up @@ -147,14 +157,15 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val b: java.lang.Short = java.lang.Short.valueOf("-22")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)
val row2 = Row(2, a, b, a, b, a, b)
val row3 = Row(3, a, a, b, a, b, a)
val row4 = Row(4, b, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", ShortType),
StructField("c2", ShortType),
StructField("c3", ShortType),
StructField("c4", ShortType),
Expand All @@ -166,15 +177,19 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val readA: Array[Byte] = Array(49.toByte, 49.toByte)
val readB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte)

val readRow1 = Row(1, null, null, null, null, null)
val readRow2 = Row(2, readB, readA, readB, readA, readB)
val readRow3 = Row(3, readA, readB, readA, readB, readA)
val readRow4 = Row(4, readA, readA, readA, readA, readA)
val readRow5 = Row(5, readB, readB, readB, readB, readB)
val binaryReadA: Array[Byte] = Array(49.toByte, 49.toByte, 0.toByte, 0.toByte, 0.toByte)
val binaryReadB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte, 0.toByte, 0.toByte)

val readRow1 = Row(1, null, null, null, null, null, null)
val readRow2 = Row(2, binaryReadA, readB, readA, readB, readA, readB)
val readRow3 = Row(3, binaryReadA, readA, readB, readA, readB, readA)
val readRow4 = Row(4, binaryReadB, readA, readA, readA, readA, readA)
val readRow5 = Row(5, binaryReadB, readB, readB, readB, readB, readB)

val readSchema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", BinaryType),
StructField("c2", BinaryType),
StructField("c3", BinaryType),
StructField("c4", BinaryType),
Expand Down Expand Up @@ -206,14 +221,15 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val b: java.lang.Integer = java.lang.Integer.valueOf("-22")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)
val row2 = Row(2, a, b, a, b, a, b)
val row3 = Row(3, a, a, b, a, b, a)
val row4 = Row(4, b, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", IntegerType),
StructField("c2", IntegerType),
StructField("c3", IntegerType),
StructField("c4", IntegerType),
Expand All @@ -225,15 +241,19 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val readA: Array[Byte] = Array(49.toByte, 49.toByte)
val readB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte)

val readRow1 = Row(1, null, null, null, null, null)
val readRow2 = Row(2, readB, readA, readB, readA, readB)
val readRow3 = Row(3, readA, readB, readA, readB, readA)
val readRow4 = Row(4, readA, readA, readA, readA, readA)
val readRow5 = Row(5, readB, readB, readB, readB, readB)
val binaryReadA: Array[Byte] = Array(49.toByte, 49.toByte, 0.toByte, 0.toByte, 0.toByte)
val binaryReadB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte, 0.toByte, 0.toByte)

val readRow1 = Row(1, null, null, null, null, null, null)
val readRow2 = Row(2, binaryReadA, readB, readA, readB, readA, readB)
val readRow3 = Row(3, binaryReadA, readA, readB, readA, readB, readA)
val readRow4 = Row(4, binaryReadB, readA, readA, readA, readA, readA)
val readRow5 = Row(5, binaryReadB, readB, readB, readB, readB, readB)

val readSchema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", BinaryType),
StructField("c2", BinaryType),
StructField("c3", BinaryType),
StructField("c4", BinaryType),
Expand Down Expand Up @@ -265,14 +285,15 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val b: java.lang.Long = java.lang.Long.valueOf("-22")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)
val row2 = Row(2, a, b, a, b, a, b)
val row3 = Row(3, a, a, b, a, b, a)
val row4 = Row(4, b, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", LongType),
StructField("c2", LongType),
StructField("c3", LongType),
StructField("c4", LongType),
Expand All @@ -284,15 +305,19 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val readA: Array[Byte] = Array(49.toByte, 49.toByte)
val readB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte)

val readRow1 = Row(1, null, null, null, null, null)
val readRow2 = Row(2, readB, readA, readB, readA, readB)
val readRow3 = Row(3, readA, readB, readA, readB, readA)
val readRow4 = Row(4, readA, readA, readA, readA, readA)
val readRow5 = Row(5, readB, readB, readB, readB, readB)
val binaryReadA: Array[Byte] = Array(49.toByte, 49.toByte, 0.toByte, 0.toByte, 0.toByte)
val binaryReadB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte, 0.toByte, 0.toByte)

val readRow1 = Row(1, null, null, null, null, null, null)
val readRow2 = Row(2, binaryReadA, readB, readA, readB, readA, readB)
val readRow3 = Row(3, binaryReadA, readA, readB, readA, readB, readA)
val readRow4 = Row(4, binaryReadB, readA, readA, readA, readA, readA)
val readRow5 = Row(5, binaryReadB, readB, readB, readB, readB, readB)

val readSchema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", BinaryType),
StructField("c2", BinaryType),
StructField("c3", BinaryType),
StructField("c4", BinaryType),
Expand All @@ -315,76 +340,6 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
}
}

// TODO: ignored
ignore("Test Convert from java.lang.Float to BYTES") {
// success
// java.lang.Float -> BYTES
compareTiDBWriteWithJDBC {
case (writeFunc, _) =>
val a: java.lang.Float = java.lang.Float.valueOf("1.1")
val b: java.lang.Float = java.lang.Float.valueOf("-2.2")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c2", FloatType),
StructField("c3", FloatType),
StructField("c4", FloatType),
StructField("c5", FloatType),
StructField("c6", FloatType)
)
)

dropTable()
createTable()

// insert rows
writeFunc(List(row1, row2, row3, row4, row5), schema, None)
compareTiDBSelectWithJDBC(Seq(row1, row2, row3, row4, row5), schema)
}
}

// TODO: java.sql.BatchUpdateException: Data truncation: Data too long for column 'c1' at row 1
ignore("Test Convert from java.lang.Double to BYTES") {
// success
// java.lang.Double -> BYTES
compareTiDBWriteWithJDBC {
case (writeFunc, _) =>
val a: java.lang.Double = java.lang.Double.valueOf("1.1")
val b: java.lang.Double = java.lang.Double.valueOf("-2.2")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c2", DoubleType),
StructField("c3", DoubleType),
StructField("c4", DoubleType),
StructField("c5", DoubleType),
StructField("c6", DoubleType)
)
)

dropTable()
createTable()

// insert rows
writeFunc(List(row1, row2, row3, row4, row5), schema, None)
compareTiDBSelectWithJDBC(Seq(row1, row2, row3, row4, row5), schema)
}
}

test("Test Convert from String to BYTES") {
// success
// java.lang.String -> BYTES
Expand All @@ -394,14 +349,15 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val b: java.lang.String = new java.lang.String("-22")

val row1 = Row(1, null, null, null, null, null, null)
val row2 = Row(2, b, a, b, a, b)
val row3 = Row(3, a, b, a, b, a)
val row4 = Row(4, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b)
val row2 = Row(2, a, b, a, b, a, b)
val row3 = Row(3, a, a, b, a, b, a)
val row4 = Row(4, b, a, a, a, a, a)
val row5 = Row(5, b, b, b, b, b, b)

val schema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", StringType),
StructField("c2", StringType),
StructField("c3", StringType),
StructField("c4", StringType),
Expand All @@ -413,15 +369,19 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
val readA: Array[Byte] = Array(49.toByte, 49.toByte)
val readB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte)

val readRow1 = Row(1, null, null, null, null, null)
val readRow2 = Row(2, readB, readA, readB, readA, readB)
val readRow3 = Row(3, readA, readB, readA, readB, readA)
val readRow4 = Row(4, readA, readA, readA, readA, readA)
val readRow5 = Row(5, readB, readB, readB, readB, readB)
val binaryReadA: Array[Byte] = Array(49.toByte, 49.toByte, 0.toByte, 0.toByte, 0.toByte)
val binaryReadB: Array[Byte] = Array(45.toByte, 50.toByte, 50.toByte, 0.toByte, 0.toByte)

val readRow1 = Row(1, null, null, null, null, null, null)
val readRow2 = Row(2, binaryReadA, readB, readA, readB, readA, readB)
val readRow3 = Row(3, binaryReadA, readA, readB, readA, readB, readA)
val readRow4 = Row(4, binaryReadB, readA, readA, readA, readA, readA)
val readRow5 = Row(5, binaryReadB, readB, readB, readB, readB, readB)

val readSchema = StructType(
List(
StructField("i", IntegerType),
StructField("c1", BinaryType),
StructField("c2", BinaryType),
StructField("c3", BinaryType),
StructField("c4", BinaryType),
Expand All @@ -445,6 +405,8 @@ class ToBytesSuite extends BaseDataSourceTest("test_data_type_convert_to_bytes")
}

// TODO: test following types
// java.lang.Float
// java.lang.Double
// java.sql.Date
// java.math.BigDecimal
// java.sql.Timestamp
Expand Down
Loading

0 comments on commit f8de3db

Please sign in to comment.