Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge e8b45b4 into 923dab1
  • Loading branch information
shardul-cr7 committed Jan 6, 2019
2 parents 923dab1 + e8b45b4 commit b4e4e24
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 1 deletion.
Expand Up @@ -333,6 +333,37 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
sql(s"DROP DATAMAP IF EXISTS $datamapName ON TABLE $longStringTable")
}

test("creating datamap with long string column selected and loading data should be success") {

sql(s"drop table if exists $longStringTable")
val datamapName = "pre_agg_dm"
sql(
s"""
| CREATE TABLE if not exists $longStringTable(
| id INT, name STRING, description STRING, address STRING, note STRING
| ) STORED BY 'carbondata'
| TBLPROPERTIES('LONG_STRING_COLUMNS'='description, note', 'SORT_COLUMNS'='name')
|""".stripMargin)

sql(
s"""
| CREATE DATAMAP $datamapName ON TABLE $longStringTable
| USING 'preaggregate'
| AS SELECT id,description,note,count(*) FROM $longStringTable
| GROUP BY id,description,note
|""".
stripMargin)

sql(
s"""
| LOAD DATA LOCAL INPATH '$inputFile' INTO TABLE $longStringTable
| OPTIONS('header'='false')
""".stripMargin)

checkAnswer(sql(s"select count(*) from $longStringTable"), Row(1000))
sql(s"drop table if exists $longStringTable")
}

test("create table with varchar column and complex column") {
sql("DROP TABLE IF EXISTS varchar_complex_table")
sql("""
Expand Down
Expand Up @@ -110,7 +110,32 @@ case class PreAggregateTableHelper(
// Datamap table name and columns are automatically added prefix with parent table name
// in carbon. For convenient, users can type column names same as the ones in select statement
// when config dmproperties, and here we update column names with prefix.
val longStringColumn = tableProperties.get(CarbonCommonConstants.LONG_STRING_COLUMNS)
// If longStringColumn is not present in dm properties then we take long_string_columns from
// the parent table.
var longStringColumn = tableProperties.get(CarbonCommonConstants.LONG_STRING_COLUMNS)
if (longStringColumn.isEmpty) {
val longStringColumnInParents = parentTable.getTableInfo.getFactTable.getTableProperties
.asScala
.getOrElse(CarbonCommonConstants.LONG_STRING_COLUMNS, "").split(",").map(_.trim)
val varcharDatamapFields = scala.collection.mutable.ArrayBuffer.empty[String]
fieldRelationMap foreach (fields => {
val aggFunc = fields._2.aggregateFunction
val relationList = fields._2.columnTableRelationList
if (aggFunc.isEmpty && relationList.size == 1) {
relationList.foreach(rel => {
rel.foreach(col => {
if (longStringColumnInParents.contains(col.parentColumnName)) {
varcharDatamapFields += col.parentColumnName
}
})
})
}
})
if (!varcharDatamapFields.isEmpty) {
longStringColumn = Option(varcharDatamapFields.mkString(","))
}
}

if (longStringColumn != None) {
val fieldNames = fields.map(_.column)
val newLongStringColumn = longStringColumn.get.split(",").map(_.trim).map{ colName =>
Expand Down

0 comments on commit b4e4e24

Please sign in to comment.