Skip to content

Commit

Permalink
[CARBONDATA-2866] Block schema in external table
Browse files Browse the repository at this point in the history 
This PR blocks schema in CREATE EXTERNAL TABLE

This closes #2645
  • Loading branch information
@jackylk @ravipesala
jackylk authored and ravipesala committed Sep 5, 2018
1 parent b611a86 commit af2c469
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 42 deletions.
19 changes: 19 additions & 0 deletions ...est/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
View file
Expand Up @@ -90,6 +90,25 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
}
}

test("create external table with specified schema") {
assert(new File(originDataPath).exists())
sql("DROP TABLE IF EXISTS source")
val ex = intercept[AnalysisException] {
sql(
s"""
|CREATE EXTERNAL TABLE source (key INT)
|STORED BY 'carbondata'
|LOCATION '$storeLocation/origin'
""".stripMargin)
}
assert(ex.message.contains("Schema may not be specified for external table"))

sql("DROP TABLE IF EXISTS source")

// DROP TABLE should not delete data
assert(new File(originDataPath).exists())
}

test("create external table with empty folder") {
val exception = intercept[AnalysisException] {
sql(
Expand Down
36 changes: 19 additions & 17 deletions ...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
View file
Expand Up @@ -278,16 +278,18 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
s"""
| CREATE EXTERNAL TABLE sdkOutputTable
| STORED BY 'carbondata'
| LOCATION '$writerPath'
""".stripMargin)

sql("drop table if exists t1")
sql("create table if not exists t1 (name string, age int, height double) STORED BY 'org.apache.carbondata.format'")
var i =0;
var i =0
while (i<50){
sql (s"""insert into t1 values ("aaaaa", 12, 20)""").show(200,false)
i = i+1;
i = i+1
}
checkAnswer(sql("select count(*) from t1"),Seq(Row(50)))
val one = Future {
Expand Down Expand Up @@ -400,7 +402,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand All @@ -413,7 +415,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS sdkOutputTable")
// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand All @@ -425,7 +427,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS sdkOutputTable")
// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand All @@ -437,7 +439,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand All @@ -464,7 +466,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand All @@ -487,7 +489,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand Down Expand Up @@ -520,7 +522,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand Down Expand Up @@ -558,7 +560,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with partition
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand Down Expand Up @@ -737,7 +739,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

// with schema
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(age int) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)

Expand Down Expand Up @@ -2260,7 +2262,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType date, course_details struct<course_struct_course_time: date>) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(java.sql.Date.valueOf("1970-04-12"), Row(java.sql.Date.valueOf("1970-01-11")))))
Expand Down Expand Up @@ -2306,7 +2308,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time: timestamp>) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
Expand Down Expand Up @@ -2353,7 +2355,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time: timestamp>) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
Expand Down
28 changes: 14 additions & 14 deletions ...bondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
View file
Expand Up @@ -180,7 +180,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
Expand Down Expand Up @@ -241,7 +241,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(true, null, null))))
Expand Down Expand Up @@ -271,7 +271,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row("abc", null, null))))
Expand Down Expand Up @@ -301,7 +301,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(null, 10, null))))
Expand Down Expand Up @@ -331,7 +331,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(null))))
Expand Down Expand Up @@ -398,7 +398,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row("VAL_0"))))
Expand Down Expand Up @@ -434,7 +434,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
sql("select * from sdkOutputTable").show(false)
Expand Down Expand Up @@ -472,7 +472,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
Expand Down Expand Up @@ -505,7 +505,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
sql("describe formatted sdkOutputTable").show(false)
Expand Down Expand Up @@ -564,7 +564,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
Expand Down Expand Up @@ -659,7 +659,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
sql("describe formatted sdkOutputTable").show(false)
Expand Down Expand Up @@ -700,7 +700,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkExistence(sql("select * from sdkOutputTable"), true, "1010.00")
Expand Down Expand Up @@ -738,7 +738,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id struct<b:decimal(4,3)>) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkExistence(sql("select * from sdkOutputTable"), true, "32.00")
Expand Down Expand Up @@ -784,7 +784,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest with Bef
writer.write(record)
writer.close()
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable(id struct<b:decimal(4,3)>) STORED BY
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkExistence(sql("select * from sdkOutputTable"), true, "32.0")
Expand Down
9 changes: 2 additions & 7 deletions integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
View file
Expand Up @@ -324,9 +324,7 @@ object CarbonSource {
tableDesc.copy(storage = updatedFormat)
} else {
val tableInfo = CarbonUtil.convertGsonToTableInfo(properties.asJava)
val isExternal = properties.getOrElse("isExternal", "false")
val isTransactionalTable = properties.getOrElse("isTransactional", "true")
.contains("true")
val isTransactionalTable = properties.getOrElse("isTransactional", "true").contains("true")
tableInfo.setTransactionalTable(isTransactionalTable)
if (isTransactionalTable && !metaStore.isReadFromHiveMetaStore) {
// save to disk
Expand All @@ -350,10 +348,7 @@ object CarbonSource {
query: Option[LogicalPlan]): Map[String, String] = {
val model = createTableInfoFromParams(properties, dataSchema, identifier, query, sparkSession)
val tableInfo: TableInfo = TableNewProcessor(model)
val isExternal = properties.getOrElse("isExternal", "false")
val isTransactionalTable = properties.getOrElse("isTransactional", "true")
.contains("true")
val tablePath = properties.getOrElse("path", "")
val isTransactionalTable = properties.getOrElse("isTransactional", "true").contains("true")
tableInfo.setTablePath(identifier.getTablePath)
tableInfo.setTransactionalTable(isTransactionalTable)
tableInfo.setDatabaseName(identifier.getDatabaseName)
Expand Down
18 changes: 14 additions & 4 deletions integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
View file
Expand Up @@ -17,6 +17,7 @@

package org.apache.spark.sql.parser

import scala.collection.JavaConverters._
import scala.collection.mutable

import org.antlr.v4.runtime.tree.TerminalNode
Expand Down Expand Up @@ -127,6 +128,16 @@ object CarbonSparkSqlParserUtil {
var isTransactionalTable: Boolean = true

val tableInfo = if (external) {
if (fields.nonEmpty) {
// user provided schema for this external table, this is not allow currently
// see CARBONDATA-2866
operationNotAllowed(
"Schema may not be specified for external table", columns)
}
if (partitionByStructFields.nonEmpty) {
operationNotAllowed(
"Partition is not supported for external table", partitionColumns)
}
// read table info from schema file in the provided table path
// external table also must convert table name to lower case
val identifier = AbsoluteTableIdentifier.from(
Expand All @@ -142,15 +153,14 @@ object CarbonSparkSqlParserUtil {
isTransactionalTable = false
SchemaReader.inferSchema(identifier, false)
}
}
else {
} else {
SchemaReader.getTableInfo(identifier)
}
}
catch {
} catch {
case e: Throwable =>
operationNotAllowed(s"Invalid table path provided: ${ tablePath.get } ", tableHeader)
}

// set "_external" property, so that DROP TABLE will not delete the data
if (provider.equalsIgnoreCase("'carbonfile'")) {
table.getFactTable.getTableProperties.put("_filelevelformat", "true")
Expand Down

0 comments on commit af2c469

Please sign in to comment.