Skip to content
Permalink
Browse files
[CARBONDATA-4273] Fix Cannot create external table with partitions
Why is this PR needed?
Create partition table with location fails with unsupported message.

What changes were proposed in this PR?
This scenario works in cluster mode. This check can be moved in local
mode also and partition table can be created with table with location

Does this PR introduce any user interface change?
No

Is any new testcase added?
Yes

This closes #4211
  • Loading branch information
Indhumathi27 authored and akashrn5 committed Sep 1, 2021
1 parent bdc9484 commit 42f69827e0a577b6128417104c0a49cd5bf21ad7
Showing 2 changed files with 73 additions and 13 deletions.
@@ -215,10 +215,6 @@ object CarbonSparkSqlParserUtil {
table.identifier.table.toLowerCase()
)
val tableInfo = if (isExternal) {
if (partitionColumnNames.nonEmpty) {
throw new MalformedCarbonCommandException(
"Partition is not supported for external table")
}
// read table info from schema file in the provided table path
val tableInfo = {
try {
@@ -19,7 +19,8 @@ package org.apache.carbondata.spark.testsuite.createTable

import java.io.File

import org.apache.spark.sql.{AnalysisException, CarbonEnv}
import org.apache.commons.io.FileUtils
import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll

@@ -76,19 +77,22 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
|STORED AS carbondata
|LOCATION '$storeLocation/origin'
""".stripMargin)
checkAnswer(sql("SELECT count(*) from source"), sql("SELECT count(*) from origin"))
verifyResult()
}
}

checkExistence(sql("describe formatted source"), true, storeLocation + "/origin")
private def verifyResult(): Unit = {
checkAnswer(sql("SELECT count(*) from source"), sql("SELECT count(*) from origin"))

val carbonTable = CarbonEnv.getCarbonTable(None, "source")(sqlContext.sparkSession)
assert(carbonTable.isExternalTable)
checkExistence(sql("describe formatted source"), true, storeLocation + "/origin")

sql("DROP TABLE IF EXISTS source")
val carbonTable = CarbonEnv.getCarbonTable(None, "source")(sqlContext.sparkSession)
assert(carbonTable.isExternalTable)

// DROP TABLE should not delete data
assert(new File(originDataPath).exists())
sql("DROP TABLE IF EXISTS source")

}
// DROP TABLE should not delete data
assert(new File(originDataPath).exists())
}

ignore("create external table with specified schema") {
@@ -146,4 +150,64 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select * from rstest1"), sql("select * from rsext"))
}

test("create external table and non-external table on partition table with location") {
sql("set hive.exec.dynamic.partition.mode=nonstrict")
sql("drop table if exists origin")
// create carbon table and insert data
sql("CREATE TABLE origin(a int, b string) partitioned by (c string) STORED AS carbondata")
sql("INSERT INTO origin select 100,'spark','test1'")
sql("INSERT INTO origin select 200,'hive','test2'")

// test external table with partition by with location
sql("drop table if exists source")
sql(
s"""
|CREATE EXTERNAL TABLE source(a int, b string) partitioned by (c string)
|stored as carbondata
|LOCATION '$storeLocation/origin'
""".stripMargin)
verifyResult()

// test table with partition by with location
sql("drop table if exists source")
sql(
s"""
|CREATE TABLE source(a int, b string) partitioned by (c string)
|stored as carbondata
|LOCATION '$storeLocation/origin'
""".stripMargin)
FileUtils.deleteDirectory(new File( s"$storeLocation/origin1"))
val newStoreLocation = s"$storeLocation/origin1"
FileUtils.copyDirectory(new File(s"$storeLocation/origin/c=test1"), new File(newStoreLocation))
verifyResult()

// test without any schema file in location specified
sql("drop table if exists source")
sql(
s"""
|CREATE TABLE source(a int, b string) partitioned by (c string)
|stored as carbondata
|LOCATION '$newStoreLocation'
""".stripMargin)
checkAnswer(sql("select * from source"), Seq(Row(100, "spark", "test1")))
sql("drop table if exists source")

// test with empty directory
FileUtils.deleteDirectory(new File( s"$storeLocation/origin1"))
sql("drop table if exists source")
sql(
s"""
|CREATE TABLE source(a int, b string) partitioned by (c string)
|stored as carbondata
|LOCATION '$newStoreLocation'
""".stripMargin)
val exception = intercept[Exception] {
sql("select * from source").show(false)
}
assert(exception.getMessage.contains("No Index files are present in the table location"))
sql("INSERT INTO source select 100,'spark','test1'")
checkAnswer(sql("select * from source"), Seq(Row(100, "spark", "test1")))
sql("drop table if exists source")
}

}

0 comments on commit 42f6982

Please sign in to comment.