Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CARBONDATA-3423] Validate dictionary for binary data type #3271

Closed
wants to merge 3 commits into from
Closed

[CARBONDATA-3423] Validate dictionary for binary data type #3271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bd4662e
[CARBONDATA-3423] Validate dictionary for binary data type
xubo245 Jun 10, 2019
89361e9
doc
xubo245 Jun 10, 2019
4242877
remove dictionary_enable
xubo245 Jun 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/ddl-of-carbondata.md
View file
Open in desktop
Expand Up @@ -126,7 +126,7 @@ CarbonData DDL statements are documented here,which includes:
```

**NOTE**:
* Dictionary Include/Exclude for complex child columns is not supported.
* Dictionary Include/Exclude for complex child columns is not supported. Dictionary Include doesn't support binary data type.
* Dictionary is global. Except global dictionary, there are local dictionary and non-dictionary in CarbonData.

- ##### Local Dictionary Configuration
Expand Down
57 changes: 57 additions & 0 deletions ...t/scala/org/apache/carbondata/integration/spark/testsuite/binary/TestBinaryDataType.scala
View file
Open in desktop
Expand Up @@ -158,6 +158,63 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
assert(true)
}


test("Unsupport DICTIONARY_INCLUDE for binary") {

sql("DROP TABLE IF EXISTS binaryTable")
val exception = intercept[MalformedCarbonCommandException] {
sql(
"""
| CREATE TABLE binaryTable(
| id int,
| name string,
| city string,
| age int,
| binaryField binary)
| STORED BY 'carbondata'
| tblproperties('dictionary_include'='binaryField')
""".stripMargin)
}
assert(exception.getMessage.contains(
"DICTIONARY_INCLUDE is unsupported for binary data type column: binaryfield"))
}

test("Unsupport DICTIONARY_INCLUDE for binary, multiple column") {

sql("DROP TABLE IF EXISTS binaryTable")
val exception = intercept[MalformedCarbonCommandException] {
sql(
"""
| CREATE TABLE binaryTable(
| id int,
| name string,
| city string,
| age int,
| binaryField binary)
| STORED BY 'carbondata'
| tblproperties('dictionary_include'='name,binaryField')
""".stripMargin)
}
assert(exception.getMessage.contains(
"DICTIONARY_INCLUDE is unsupported for binary data type column: binaryfield"))
}

test("Supports DICTIONARY_EXCLUDE for binary") {
sql("DROP TABLE IF EXISTS binaryTable")
sql(
"""
| CREATE TABLE binaryTable(
| id int,
| name string,
| city string,
| age int,
| binaryField binary)
| STORED BY 'org.apache.carbondata.format'
| tblproperties('DICTIONARY_EXCLUDE'='binaryField')
""".stripMargin)
assert(true)
}

test("Unsupport inverted_index for binary") {
sql("DROP TABLE IF EXISTS binaryTable")
val exception = intercept[MalformedCarbonCommandException] {
Expand Down
8 changes: 7 additions & 1 deletion ...ration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
View file
Open in desktop
Expand Up @@ -36,7 +36,7 @@ import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandExcepti
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.exception.InvalidConfigurationException
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.PartitionInfo
import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
Expand Down Expand Up @@ -841,6 +841,12 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
}
val rangeField = fields.find(_.column.equalsIgnoreCase(distIncludeCol.trim))
if ("binary".equalsIgnoreCase(rangeField.get.dataType.get)) {
throw new MalformedCarbonCommandException(
"DICTIONARY_INCLUDE is unsupported for binary data type column: " +
distIncludeCol.trim)
}
if (varcharCols.exists(x => x.equalsIgnoreCase(distIncludeCol.trim))) {
throw new MalformedCarbonCommandException(
"DICTIONARY_INCLUDE is unsupported for long string datatype column: " +
Expand Down