From d79ea87829ea18fda08bb47fc0f6694c73f2d78f Mon Sep 17 00:00:00 2001 From: ngerstle Date: Fri, 9 Dec 2016 10:17:20 +0100 Subject: [PATCH 1/4] Update RedshiftJDBCWrapper.scala java.sql.Types.BIGINT => LongType in all cases for Redshift. Redshift has no unsigned BigInt, and thus BigInt maps to Long without loss of resolution. --- .../com/databricks/spark/redshift/RedshiftJDBCWrapper.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/databricks/spark/redshift/RedshiftJDBCWrapper.scala b/src/main/scala/com/databricks/spark/redshift/RedshiftJDBCWrapper.scala index dc72dccf..4253ac79 100644 --- a/src/main/scala/com/databricks/spark/redshift/RedshiftJDBCWrapper.scala +++ b/src/main/scala/com/databricks/spark/redshift/RedshiftJDBCWrapper.scala @@ -301,7 +301,7 @@ private[redshift] class JDBCWrapper { val answer = sqlType match { // scalastyle:off case java.sql.Types.ARRAY => null - case java.sql.Types.BIGINT => if (signed) { LongType } else { DecimalType(20,0) } + case java.sql.Types.BIGINT => LongType case java.sql.Types.BINARY => BinaryType case java.sql.Types.BIT => BooleanType // @see JdbcDialect for quirks case java.sql.Types.BLOB => BinaryType From 6a16c4b176219a44dca8b7c3e04e5042019b7fc9 Mon Sep 17 00:00:00 2001 From: ngerstle Date: Fri, 9 Dec 2016 15:02:57 +0100 Subject: [PATCH 2/4] Adding regression tests for LongType errors Errors/unusually behavior is associated with BigInts. Specifically, a count(column) query sometimes returns doubles rather than longs, as would be expected from the count function (which only provides signed BigInts). --- .../spark/redshift/RedshiftReadSuite.scala | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala b/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala index ec2779ab..89d275ef 100644 --- a/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala +++ b/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala @@ -225,6 +225,8 @@ class RedshiftReadSuite extends IntegrationSuiteBase { } } + + test("read records containing escaped characters") { withTempRedshiftTable("records_with_escaped_characters") { tableName => conn.createStatement().executeUpdate( @@ -245,4 +247,22 @@ class RedshiftReadSuite extends IntegrationSuiteBase { .load() assert(df.schema.fields(0).dataType === LongType) } + + + test("read result of count() query (a BigInt) returned as LongType (regression for #310)") { + val df = read + .option("query", s"select count(testbool) as c from $test_table") + .load() + assert(df.schema.fields(0).dataType === LongType) + } + + + test("read result returning a BigInt becomes a LongType (regression for #311)") { + val df = read + .option("query", s"select testlong::BigInt as c from $test_table") + .load() + assert(df.schema.fields(0).dataType === LongType) + } + + } From 3809feab53499ea80df3fa968b4b80ba5bfdfe6a Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 5 Jan 2017 12:13:01 -0800 Subject: [PATCH 3/4] Remove excess whitespace from new test cases. --- .../com/databricks/spark/redshift/RedshiftReadSuite.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala b/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala index 89d275ef..84636a9f 100644 --- a/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala +++ b/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala @@ -224,8 +224,6 @@ class RedshiftReadSuite extends IntegrationSuiteBase { conn.commit() } } - - test("read records containing escaped characters") { withTempRedshiftTable("records_with_escaped_characters") { tableName => @@ -248,7 +246,6 @@ class RedshiftReadSuite extends IntegrationSuiteBase { assert(df.schema.fields(0).dataType === LongType) } - test("read result of count() query (a BigInt) returned as LongType (regression for #310)") { val df = read .option("query", s"select count(testbool) as c from $test_table") @@ -256,13 +253,10 @@ class RedshiftReadSuite extends IntegrationSuiteBase { assert(df.schema.fields(0).dataType === LongType) } - test("read result returning a BigInt becomes a LongType (regression for #311)") { val df = read .option("query", s"select testlong::BigInt as c from $test_table") .load() assert(df.schema.fields(0).dataType === LongType) } - - } From de15c0c373779dbeacf89db372051c462e3c981a Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 5 Jan 2017 12:13:21 -0800 Subject: [PATCH 4/4] More whitespace fixes. --- .../scala/com/databricks/spark/redshift/RedshiftReadSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala b/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala index 84636a9f..d3d489ee 100644 --- a/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala +++ b/src/it/scala/com/databricks/spark/redshift/RedshiftReadSuite.scala @@ -224,7 +224,7 @@ class RedshiftReadSuite extends IntegrationSuiteBase { conn.commit() } } - + test("read records containing escaped characters") { withTempRedshiftTable("records_with_escaped_characters") { tableName => conn.createStatement().executeUpdate(