From 2b6789ee5464f90f43ecac3654e58424257eaa29 Mon Sep 17 00:00:00 2001 From: m00258959 Date: Mon, 5 Nov 2018 15:45:46 +0530 Subject: [PATCH] Fixed select query failure due to stale carbonTable in dataMapFactory class --- .../core/datamap/DataMapStoreManager.java | 6 ++ .../core/datamap/dev/DataMapFactory.java | 4 + .../test/resources/vardhandaterestruct.csv | 99 +++++++++++++++++++ .../SparkCarbonDataSourceTest.scala | 18 ++++ 4 files changed, 127 insertions(+) create mode 100644 integration/spark-datasource/src/test/resources/vardhandaterestruct.csv diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java index 75290d3a434..baf4739d87d 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java @@ -345,6 +345,12 @@ public TableDataMap getDataMap(CarbonTable table, DataMapSchema dataMapSchema) { if (dataMap == null) { throw new RuntimeException("Datamap does not exist"); } + // This is done to handle the scenario of stale cache because of which schema mismatch + // exception can be thrown. Scenario: In case of carbondata used through FileFormat API, + // once a table is dropped and recreated with the same name again then because the dataMap + // contains the stale carbon table schema mismatch exception is thrown. To avoid such scenarios + // it is always better to update the carbon table object retrieved + dataMap.getDataMapFactory().setCarbonTable(table); return dataMap; } diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java index de8dc5811d0..ee7914df7f3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java @@ -54,6 +54,10 @@ public CarbonTable getCarbonTable() { return carbonTable; } + public void setCarbonTable(CarbonTable carbonTable) { + this.carbonTable = carbonTable; + } + public DataMapSchema getDataMapSchema() { return dataMapSchema; } diff --git a/integration/spark-datasource/src/test/resources/vardhandaterestruct.csv b/integration/spark-datasource/src/test/resources/vardhandaterestruct.csv new file mode 100644 index 00000000000..daa7c381ab6 --- /dev/null +++ b/integration/spark-datasource/src/test/resources/vardhandaterestruct.csv @@ -0,0 +1,99 @@ +1AA1,1,8RAM size,4,Chinese,guangzhou,2738,2014-07-01 12:07:28,2014-07-01 12:07:28,25 +1AA10,10,4RAM size,4,Chinese,wuhan,1714,2014-07-02 12:07:28,2014-07-02 12:07:28,26 +1AA100,100,7RAM size,6,Chinese,yichang,1271,2014-07-03 12:07:28,2014-07-03 12:07:28,27 +1AA1000,1000,5RAM size,3,Chinese,changsha,692,2014-07-04 12:07:28,2014-07-04 12:07:28,28 +1AA10000,10000,1RAM size,1,Chinese,xiangtan,2175,2015-07-05 12:07:28,2015-07-05 12:07:28,29 +1AA100000,100000,4RAM size,6,Chinese,yichang,136,2015-07-06 12:07:28,2015-07-06 12:07:28,30 +1AA1000000,1000000,4RAM size,6,Chinese,xiangtan,1600,2015-07-07 12:07:28,2015-07-07 12:07:28,31 +1AA100001,100001,7RAM size,7,Chinese,wuhan,505,2014-07-08 12:07:28,2014-07-08 12:07:28,32 +1AA100002,100002,0RAM size,3,Chinese,zhuzhou,1341,2015-07-09 12:07:28,2015-07-09 12:07:28,33 +1AA100003,100003,7RAM size,3,Chinese,zhuzhou,2239,2015-07-10 12:07:28,2015-07-10 12:07:28,34 +1AA100004,100004,8RAM size,2,Chinese,wuhan,2970,2014-07-11 12:07:28,2014-07-11 12:07:28,35 +1AA100005,100005,0RAM size,1,Chinese,changsha,2593,2015-07-12 12:07:28,2015-07-12 12:07:28,36 +1AA100006,100006,4RAM size,2,Chinese,changsha,2572,2015-07-13 12:07:28,2015-07-13 12:07:28,37 +1AA100007,100007,4RAM size,3,Chinese,xiangtan,1991,2015-07-14 12:07:28,2015-07-14 12:07:28,38 +1AA100008,100008,0RAM size,1,Chinese,wuhan,1442,2015-07-15 12:07:28,2015-07-15 12:07:28,39 +1AA100009,100009,4RAM size,3,Chinese,xiangtan,1841,2014-07-16 12:07:28,2014-07-16 12:07:28,40 +1AA10001,10001,6RAM size,5,Chinese,xiangtan,298,2015-07-17 12:07:28,2015-07-17 12:07:28,41 +1AA100010,100010,0RAM size,6,Chinese,guangzhou,79,2015-07-18 12:07:28,2015-07-18 12:07:28,42 +1AA100011,100011,1RAM size,1,Chinese,xiangtan,202,2015-07-19 12:07:28,2015-07-19 12:07:28,43 +1AA100012,100012,6RAM size,2,Chinese,xiangtan,568,2015-07-20 12:07:28,2015-07-20 12:07:28,44 +1AA100013,100013,8RAM size,6,Chinese,shenzhen,355,2015-07-21 12:07:28,2015-07-21 12:07:28,45 +1AA100014,100014,7RAM size,3,Chinese,changsha,151,2015-07-22 12:07:28,2015-07-22 12:07:28,46 +1AA100015,100015,3RAM size,1,Chinese,changsha,2863,2015-07-23 12:07:28,2015-07-23 12:07:28,47 +1AA100016,100016,8RAM size,3,Chinese,xiangtan,1873,2015-07-24 12:07:28,2015-07-24 12:07:28,48 +1AA100017,100017,5RAM size,3,Chinese,guangzhou,2205,2015-07-25 12:07:28,2015-07-25 12:07:28,49 +1AA100018,100018,4RAM size,4,Chinese,yichang,441,2015-07-26 12:07:28,2015-07-26 12:07:28,50 +1AA100019,100019,0RAM size,3,Chinese,changsha,2194,2015-07-27 12:07:28,2015-07-27 12:07:28,51 +1AA10002,10002,8RAM size,5,Chinese,yichang,2972,2015-07-28 12:07:28,2015-07-28 12:07:28,52 +1AA100020,100020,1RAM size,7,Chinese,shenzhen,256,2015-07-29 12:07:28,2015-07-29 12:07:28,53 +1AA100021,100021,0RAM size,6,Chinese,changsha,1778,2015-07-30 12:07:28,2015-07-30 12:07:28,54 +1AA100022,100022,3RAM size,5,Chinese,guangzhou,1999,2015-07-31 12:07:28,2015-07-31 12:07:28,55 +1AA100023,100023,8RAM size,3,Chinese,yichang,2194,2015-08-01 12:07:28,2015-08-01 12:07:28,56 +1AA100024,100024,0RAM size,2,Chinese,wuhan,2483,2015-08-02 12:07:28,2015-08-02 12:07:28,57 +1AA100025,100025,7RAM size,1,Chinese,wuhan,1724,2015-08-03 12:07:28,2015-08-03 12:07:28,58 +1AA100026,100026,6RAM size,7,Chinese,guangzhou,1768,2015-08-04 12:07:28,2015-08-04 12:07:28,59 +1AA100027,100027,3RAM size,4,Chinese,zhuzhou,2436,2015-08-05 12:07:28,2015-08-05 12:07:28,60 +1AA100028,100028,0RAM size,5,Chinese,shenzhen,2849,2015-08-06 12:07:28,2015-08-06 12:07:28,61 +1AA100029,100029,4RAM size,5,Chinese,changsha,1691,2015-08-07 12:07:28,2015-08-07 12:07:28,62 +1AA10003,10003,9RAM size,4,Chinese,xiangtan,2071,2015-08-08 12:07:28,2015-08-08 12:07:28,63 +1AA100030,100030,1RAM size,3,Chinese,guangzhou,1333,2015-08-09 12:07:28,2015-08-09 12:07:28,64 +1AA100031,100031,3RAM size,1,Chinese,xiangtan,1080,2015-08-10 12:07:28,2015-08-10 12:07:28,65 +1AA100032,100032,3RAM size,7,Chinese,shenzhen,1053,2015-08-11 12:07:28,2015-08-11 12:07:28,66 +1AA100033,100033,7RAM size,6,Chinese,yichang,760,2015-08-12 12:07:28,2015-08-12 12:07:28,67 +1AA100034,100034,6RAM size,6,Chinese,changsha,2061,2015-08-13 12:07:28,2015-08-13 12:07:28,68 +1AA100035,100035,6RAM size,7,Chinese,shenzhen,2142,2015-08-14 12:07:28,2015-08-14 12:07:28,69 +1AA100036,100036,9RAM size,5,Chinese,changsha,2224,2015-08-15 12:07:28,2015-08-15 12:07:28,70 +1AA100037,100037,9RAM size,6,Chinese,changsha,1015,2015-08-16 12:07:28,2015-08-16 12:07:28,71 +1AA100038,100038,8RAM size,3,Chinese,xiangtan,1229,2015-08-17 12:07:28,2015-08-17 12:07:28,72 +1AA100039,100039,7RAM size,1,Chinese,wuhan,1750,2015-08-18 12:07:28,2015-08-18 12:07:28,73 +1AA10004,10004,4RAM size,4,Chinese,shenzhen,1717,2015-08-19 12:07:28,2015-08-19 12:07:28,74 +1AA100040,100040,1RAM size,7,Chinese,yichang,2078,2015-08-20 12:07:28,2015-08-20 12:07:28,75 +1AA100041,100041,1RAM size,1,Chinese,xiangtan,2734,2015-08-21 12:07:28,2015-08-21 12:07:28,76 +1AA100042,100042,1RAM size,5,Chinese,zhuzhou,2745,2015-08-22 12:07:28,2015-08-22 12:07:28,77 +1AA100043,100043,9RAM size,6,Chinese,yichang,571,2015-08-23 12:07:28,2015-08-23 12:07:28,78 +1AA100044,100044,9RAM size,2,Chinese,shenzhen,1697,2015-08-24 12:07:28,2015-08-24 12:07:28,79 +1AA100045,100045,4RAM size,6,Chinese,wuhan,2553,2015-08-25 12:07:28,2015-08-25 12:07:28,80 +1AA100046,100046,4RAM size,4,Chinese,wuhan,1077,2015-08-26 12:07:28,2015-08-26 12:07:28,81 +1AA100047,100047,6RAM size,1,Chinese,wuhan,1823,2015-08-27 12:07:28,2015-08-27 12:07:28,82 +1AA100048,100048,1RAM size,1,Chinese,xiangtan,2399,2015-08-28 12:07:28,2015-08-28 12:07:28,83 +1AA100049,100049,4RAM size,6,Chinese,xiangtan,2890,2015-08-29 12:07:28,2015-08-29 12:07:28,84 +1AA10005,10005,3RAM size,2,Chinese,zhuzhou,1608,2015-08-30 12:07:28,2015-08-30 12:07:28,85 +1AA100050,100050,4RAM size,1,Chinese,yichang,29,2015-08-31 12:07:28,2015-08-31 12:07:28,86 +1AA100051,100051,3RAM size,7,Chinese,xiangtan,1407,2015-09-01 12:07:28,2015-09-01 12:07:28,87 +1AA100052,100052,8RAM size,7,Chinese,zhuzhou,845,2015-09-02 12:07:28,2015-09-02 12:07:28,88 +1AA100053,100053,3RAM size,3,Chinese,zhuzhou,1655,2015-09-03 12:07:28,2015-09-03 12:07:28,89 +1AA100054,100054,9RAM size,2,Chinese,shenzhen,1368,2015-09-04 12:07:28,2015-09-04 12:07:28,90 +1AA100055,100055,4RAM size,7,Chinese,guangzhou,1728,2015-09-05 12:07:28,2015-09-05 12:07:28,91 +1AA100056,100056,0RAM size,5,Chinese,wuhan,750,2015-09-06 12:07:28,2015-09-06 12:07:28,92 +1AA100057,100057,4RAM size,6,Chinese,changsha,2288,2015-09-07 12:07:28,2015-09-07 12:07:28,93 +1AA100058,100058,3RAM size,4,Chinese,wuhan,2635,2015-09-08 12:07:28,2015-09-08 12:07:28,94 +1AA100059,100059,3RAM size,7,Chinese,xiangtan,1337,2015-09-09 12:07:28,2015-09-09 12:07:28,95 +1AA10006,10006,5RAM size,5,Chinese,wuhan,2478,2015-09-10 12:07:28,2015-09-10 12:07:28,96 +1AA100060,100060,4RAM size,4,Chinese,shenzhen,538,2015-09-11 12:07:28,2015-09-11 12:07:28,97 +1AA100061,100061,0RAM size,6,Chinese,wuhan,1407,2015-09-12 12:07:28,2015-09-12 12:07:28,98 +1AA100062,100062,6RAM size,6,Chinese,zhuzhou,2952,2015-09-13 12:07:28,2015-09-13 12:07:28,99 +1AA100063,100063,8RAM size,3,Chinese,zhuzhou,1226,2015-09-14 12:07:28,2015-09-14 12:07:28,100 +1AA100064,100064,4RAM size,7,Chinese,changsha,865,2015-09-15 12:07:28,2015-09-15 12:07:28,101 +1AA100065,100065,4RAM size,7,Chinese,changsha,901,2015-09-16 12:07:28,2015-09-16 12:07:28,102 +1AA100066,100066,1RAM size,4,Chinese,yichang,1864,2015-09-17 12:07:28,2015-09-17 12:07:28,103 +1AA100067,100067,6RAM size,7,Chinese,changsha,572,2015-09-18 12:07:28,2015-09-18 12:07:28,104 +1AA100068,100068,4RAM size,7,Chinese,xiangtan,412,2015-09-19 12:07:28,2015-09-19 12:07:28,105 +1AA100069,100069,3RAM size,5,Chinese,yichang,1491,2015-09-20 12:07:28,2015-09-20 12:07:28,106 +1AA10007,10007,2RAM size,3,Chinese,xiangtan,1350,2015-09-21 12:07:28,2015-09-21 12:07:28,107 +1AA100070,100070,9RAM size,3,Chinese,wuhan,1567,2015-09-22 12:07:28,2015-09-22 12:07:28,108 +1AA100071,100071,2RAM size,5,Chinese,changsha,1973,2015-09-23 12:07:28,2015-09-23 12:07:28,109 +1AA100072,100072,9RAM size,7,Chinese,xiangtan,448,2015-09-24 12:07:28,2015-09-24 12:07:28,110 +1AA100073,100073,3RAM size,2,Chinese,shenzhen,2488,2015-09-25 12:07:28,2015-09-25 12:07:28,111 +1AA100074,100074,3RAM size,7,Chinese,shenzhen,907,2015-09-26 12:07:28,2015-09-26 12:07:28,112 +1AA100075,100075,5RAM size,6,Chinese,guangzhou,2507,2015-09-27 12:07:28,2015-09-27 12:07:28,113 +1AA100076,100076,4RAM size,7,Chinese,yichang,732,2015-09-28 12:07:28,2015-09-28 12:07:28,114 +1AA100077,100077,5RAM size,6,Chinese,changsha,2077,2015-09-29 12:07:28,2015-09-29 12:07:28,115 +1AA100078,100078,6RAM size,5,Chinese,wuhan,1434,2015-09-30 12:07:28,2015-09-30 12:07:28,116 +1AA100079,100079,0RAM size,6,Chinese,changsha,1098,2015-10-01 12:07:28,2015-10-01 12:07:28,117 +1AA10008,10008,4RAM size,4,Chinese,changsha,813,2015-10-02 12:07:28,2015-10-02 12:07:28,118 +1AA100080,100080,9RAM size,6,Chinese,xiangtan,954,2015-10-03 12:07:28,2015-10-03 12:07:28,119 +1AA100081,100081,8RAM size,2,Chinese,zhuzhou,613,2015-10-04 12:07:28,2015-10-04 12:07:28,120 +1AA100082,100082,9RAM size,7,Chinese,xiangtan,2348,2015-10-05 12:07:28,2015-10-05 12:07:28,121 +1AA100083,100083,3RAM size,2,Chinese,shenzhen,2192,2015-10-06 12:07:28,2015-10-06 12:07:28,122 +1AA100084,100084,4RAM size,4,Chinese,xiangtan,2826,2015-10-07 12:07:28,2015-10-07 12:07:28,123 diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala index 7564158be32..955d63a88a0 100644 --- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala +++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala @@ -1310,6 +1310,24 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll { spark.sql(s"drop table if exists t_carbn01b") } + test("test fileformat flow with drop and query on same table") { + spark.sql("drop table if exists fileformat_drop") + spark.sql("drop table if exists fileformat_drop_hive") + spark.sql("create table fileformat_drop (imei string,AMSize string,channelsId string,ActiveCountry string, Activecity string,gamePointId double,deviceInformationId double,productionDate Timestamp,deliveryDate timestamp,deliverycharge double) using carbon options('table_blocksize'='1','LOCAL_DICTIONARY_ENABLE'='TRUE','LOCAL_DICTIONARY_THRESHOLD'='1000')") + spark.sql("create table fileformat_drop_hive(imei string,deviceInformationId double,AMSize string,channelsId string,ActiveCountry string,Activecity string,gamePointId double,productionDate Timestamp,deliveryDate timestamp,deliverycharge double)row format delimited FIELDS terminated by ',' LINES terminated by '\n' stored as textfile") + val sourceFile = FileFactory.getPath(s"$resource/vardhandaterestruct.csv").toString + spark.sql(s"load data local inpath '$sourceFile' into table fileformat_drop_hive") + spark.sql("insert into fileformat_drop select imei ,deviceInformationId ,AMSize ,channelsId ,ActiveCountry ,Activecity ,gamePointId ,productionDate ,deliveryDate ,deliverycharge from fileformat_drop_hive") + assert(spark.sql("select count(*) from fileformat_drop where imei='1AA10000'").collect().length == 1) + + spark.sql("drop table if exists fileformat_drop") + spark.sql("create table fileformat_drop (imei string,deviceInformationId double,AMSize string,channelsId string,ActiveCountry string,Activecity string,gamePointId float,productionDate timestamp,deliveryDate timestamp,deliverycharge decimal(10,2)) using carbon options('table_blocksize'='1','LOCAL_DICTIONARY_ENABLE'='true','local_dictionary_threshold'='1000')") + spark.sql("insert into fileformat_drop select imei ,deviceInformationId ,AMSize ,channelsId ,ActiveCountry ,Activecity ,gamePointId ,productionDate ,deliveryDate ,deliverycharge from fileformat_drop_hive") + assert(spark.sql("select count(*) from fileformat_drop where imei='1AA10000'").collect().length == 1) + spark.sql("drop table if exists fileformat_drop") + spark.sql("drop table if exists fileformat_drop_hive") + } + override protected def beforeAll(): Unit = { drop createParquetTable