/
MVCreateTestCase.scala
1014 lines (906 loc) · 55.5 KB
/
MVCreateTestCase.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.carbondata.mv.rewrite
import java.io.File
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
drop()
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
val projectPath = new File(this.getClass.getResource("/").getPath + "../../../../../")
.getCanonicalPath.replaceAll("\\\\", "/")
val integrationPath = s"$projectPath/integration"
val resourcesPath = s"$integrationPath/spark-common-test/src/test/resources"
sql(
"""
| CREATE TABLE fact_table1 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(
"""
| CREATE TABLE fact_table2 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(
"""
| CREATE TABLE fact_table3 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table3 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table3 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(
"""
| CREATE TABLE fact_table4 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table4 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table4 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(
"""
| CREATE TABLE fact_table5 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(
"""
| CREATE TABLE fact_table6 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE fact_table6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
}
test("test create datamap with simple and same projection") {
sql("drop datamap if exists datamap1")
sql("create datamap datamap1 using 'mv' as select empname, designation from fact_table1")
val df = sql("select empname,designation from fact_table1")
val analyzed = df.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap1"))
checkAnswer(df, sql("select empname,designation from fact_table2"))
sql(s"drop datamap datamap1")
}
test("test create datamap with simple and sub projection") {
sql("drop datamap if exists datamap2")
sql("create datamap datamap2 using 'mv' as select empname, designation from fact_table1")
val df = sql("select empname from fact_table1")
val analyzed = df.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap2"))
checkAnswer(df, sql("select empname from fact_table2"))
sql(s"drop datamap datamap2")
}
test("test create datamap with simple and same projection with projection filter") {
sql("drop datamap if exists datamap3")
sql("create datamap datamap3 using 'mv' as select empname, designation from fact_table1")
val frame = sql("select empname, designation from fact_table1 where empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap3"))
checkAnswer(frame, sql("select empname, designation from fact_table2 where empname='shivani'"))
sql(s"drop datamap datamap3")
}
test("test create datamap with simple and sub projection with non projection filter") {
sql("create datamap datamap4 using 'mv' as select empname, designation from fact_table1")
val frame = sql("select designation from fact_table1 where empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap4"))
checkAnswer(frame, sql("select designation from fact_table2 where empname='shivani'"))
sql(s"drop datamap datamap4")
}
test("test create datamap with simple and sub projection with datamap filter") {
sql("create datamap datamap5 using 'mv' as select empname, designation from fact_table1 where empname='shivani'")
val frame = sql("select designation from fact_table1 where empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap5"))
checkAnswer(frame, sql("select designation from fact_table2 where empname='shivani'"))
sql(s"drop datamap datamap5")
}
test("test create datamap with simple and same projection with datamap filter ") {
sql("create datamap datamap6 using 'mv' as select empname, designation from fact_table1 where empname='shivani'")
val frame = sql("select empname,designation from fact_table1 where empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap6"))
checkAnswer(frame, sql("select empname,designation from fact_table2 where empname='shivani'"))
sql(s"drop datamap datamap6")
}
test("test create datamap with simple and same projection with datamap filter and extra query column filter") {
sql("create datamap datamap7 using 'mv' as select empname, designation from fact_table1 where empname='shivani'")
val frame = sql(
"select empname,designation from fact_table1 where empname='shivani' and designation='SA'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap7"))
checkAnswer(frame, sql("select empname,designation from fact_table2 where empname='shivani' and designation='SA'"))
sql(s"drop datamap datamap7")
}
test("test create datamap with simple and same projection with datamap filter and different column filter") {
sql("create datamap datamap8 using 'mv' as select empname, designation from fact_table1 where empname='shivani'")
val frame = sql("select empname,designation from fact_table1 where designation='SA'")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap8"))
checkAnswer(frame, sql("select empname,designation from fact_table2 where designation='SA'"))
sql(s"drop datamap datamap8")
}
test("test create datamap with simple and same projection with datamap filter on non projection column and extra column filter") {
sql("create datamap datamap9 using 'mv' as select empname, designation,deptname from fact_table1 where deptname='cloud'")
val frame = sql("select empname,designation from fact_table1 where deptname='cloud'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap9"))
checkAnswer(frame, sql("select empname,designation from fact_table2 where deptname='cloud'"))
sql(s"drop datamap datamap9")
}
test("test create datamap with simple and same projection with datamap filter on non projection column and no column filter") {
sql("create datamap datamap10 using 'mv' as select empname, designation,deptname from fact_table1 where deptname='cloud'")
val frame = sql("select empname,designation from fact_table1")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap10"))
checkAnswer(frame, sql("select empname,designation from fact_table2"))
sql(s"drop datamap datamap10")
}
test("test create datamap with simple and same projection with datamap filter on non projection column and different column filter") {
sql("create datamap datamap11 using 'mv' as select empname, designation,deptname from fact_table1 where deptname='cloud'")
val frame = sql("select empname,designation from fact_table1 where designation='SA'")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap11"))
checkAnswer(frame, sql("select empname,designation from fact_table2 where designation='SA'"))
sql(s"drop datamap datamap11")
}
test("test create datamap with simple and same group by query") {
sql("drop datamap if exists datamap12")
sql("create datamap datamap12 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
val frame = sql("select empname, sum(utilization) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap12"))
checkAnswer(frame, sql("select empname, sum(utilization) from fact_table2 group by empname"))
sql(s"drop datamap datamap12")
}
test("test create datamap with simple and sub group by query") {
sql("drop datamap if exists datamap13")
sql("create datamap datamap13 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
val frame = sql("select sum(utilization) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap13"))
checkAnswer(frame, sql("select sum(utilization) from fact_table2 group by empname"))
sql(s"drop datamap datamap13")
}
test("test create datamap with simple and sub group by query with filter on query") {
sql("drop datamap if exists datamap14")
sql("create datamap datamap14 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
val frame = sql(
"select empname,sum(utilization) from fact_table1 group by empname having empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap14"))
checkAnswer(frame, sql("select empname,sum(utilization) from fact_table2 where empname='shivani' group by empname"))
sql(s"drop datamap datamap14")
}
test("test create datamap with simple and sub group and sub projection by query with filter on query") {
sql("drop datamap if exists datamap32")
sql("create datamap datamap32 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
val frame = sql(
"select empname, sum(utilization) from fact_table1 group by empname having empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap32"))
checkAnswer(frame, sql( "select empname, sum(utilization) from fact_table2 group by empname having empname='shivani'"))
sql(s"drop datamap datamap32")
}
test("test create datamap with simple and sub group by query with filter on datamap") {
sql("create datamap datamap15 using 'mv' as select empname, sum(utilization) from fact_table1 where empname='shivani' group by empname")
val frame = sql(
"select empname,sum(utilization) from fact_table1 where empname='shivani' group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap15"))
checkAnswer(frame, sql("select empname,sum(utilization) from fact_table2 where empname='shivani' group by empname"))
sql(s"drop datamap datamap15")
}
test("test create datamap with simple and sub group by query with filter on datamap and no filter on query") {
sql("create datamap datamap16 using 'mv' as select empname, sum(utilization) from fact_table1 where empname='shivani' group by empname")
val frame = sql("select empname,sum(utilization) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap16"))
checkAnswer(frame, sql("select empname,sum(utilization) from fact_table2 group by empname"))
sql(s"drop datamap datamap16")
}
test("test create datamap with simple and same group by with expression") {
sql("create datamap datamap17 using 'mv' as select empname, sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group by empname")
val frame = sql(
"select empname, sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group" +
" by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap17"))
checkAnswer(frame, sql("select empname, sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table2 group" +
" by empname"))
sql(s"drop datamap datamap17")
}
test("test create datamap with simple and sub group by with expression") {
sql("drop datamap if exists datamap18")
sql("create datamap datamap18 using 'mv' as select empname, sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group by empname")
val frame = sql(
"select sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap18"))
checkAnswer(frame, sql("select sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table2 group by empname"))
sql(s"drop datamap datamap18")
}
test("test create datamap with simple and sub count group by with expression") {
sql("drop datamap if exists datamap19")
sql("create datamap datamap19 using 'mv' as select empname, count(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group by empname")
val frame = sql(
"select count(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap19"))
checkAnswer(frame, sql("select count(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table2 group by empname"))
sql(s"drop datamap datamap19")
}
test("test create datamap with simple and sub group by with expression and filter on query") {
sql("drop datamap if exists datamap20")
sql("create datamap datamap20 using 'mv' as select empname, sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 group by empname")
val frame = sql(
"select sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table1 where " +
"empname='shivani' group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap20"))
checkAnswer(frame, sql("select sum(CASE WHEN utilization=27 THEN deptno ELSE 0 END) from fact_table2 where " +
"empname='shivani' group by empname"))
sql(s"drop datamap datamap20")
}
test("test create datamap with simple join") {
sql("drop datamap if exists datamap21")
sql("create datamap datamap21 using 'mv' as select t1.empname as c1, t2.designation, t2.empname as c2 from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname)")
val frame = sql(
"select t1.empname as c1, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap21"))
checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2 where t1.empname = t2.empname"))
sql(s"drop datamap datamap21")
}
test("test create datamap with simple join and filter on query") {
sql("drop datamap if exists datamap22")
sql("create datamap datamap22 using 'mv' as select t1.empname, t2.designation,t2.empname from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname)")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = " +
"t2.empname and t1.empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap22"))
checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2 where t1.empname = " +
"t2.empname and t1.empname='shivani'"))
sql(s"drop datamap datamap22")
}
test("test create datamap with simple join and filter on query and datamap") {
sql("drop datamap if exists datamap23")
sql("create datamap datamap23 using 'mv' as select t1.empname, t2.designation, t2.empname from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) where t1.empname='shivani'")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = " +
"t2.empname and t1.empname='shivani'")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap23"))
checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2 where t1.empname = " +
"t2.empname and t1.empname='shivani'"))
sql(s"drop datamap datamap23")
}
test("test create datamap with simple join and filter on datamap and no filter on query") {
sql("drop datamap if exists datamap24")
sql("create datamap datamap24 using 'mv' as select t1.empname, t2.designation, t2.empname from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) where t1.empname='shivani'")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap24"))
checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2 where t1.empname = t2.empname"))
sql(s"drop datamap datamap24")
}
test("test create datamap with multiple join") {
sql("drop datamap if exists datamap25")
sql("create datamap datamap25 using 'mv' as select t1.empname as c1, t2.designation, t2.empname, t3.empname from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) inner join fact_table3 t3 on (t1.empname=t3.empname)")
val frame = sql(
"select t1.empname as c1, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap25"))
val frame1 = sql(
"select t1.empname as c1, t2.designation from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) inner join fact_table3 t3 on (t1.empname=t3.empname)")
val analyzed1 = frame1.queryExecution.analyzed
assert(verifyMVDataMap(analyzed1, "datamap25"))
checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2 where t1.empname = t2.empname"))
sql(s"drop datamap datamap25")
}
ignore("test create datamap with simple join on datamap and multi join on query") {
sql("create datamap datamap26 using 'mv' as select t1.empname, t2.designation, t2.empname from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname)")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2,fact_table3 " +
"t3 where t1.empname = t2.empname and t1.empname=t3.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap26"))
checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2,fact_table6 " +
"t3 where t1.empname = t2.empname and t1.empname=t3.empname"))
sql(s"drop datamap datamap26")
}
test("test create datamap with join with group by") {
sql("create datamap datamap27 using 'mv' as select t1.empname , t2.designation, sum(t1.utilization), sum(t2.empname) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
"where t1.empname = t2.empname group by t1.empname, t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap27"))
checkAnswer(frame, sql("select t1.empname, t2.designation, sum(t1.utilization) from fact_table4 t1,fact_table5 t2 " +
"where t1.empname = t2.empname group by t1.empname, t2.designation"))
sql(s"drop datamap datamap27")
}
test("test create datamap with join with group by and sub projection") {
sql("drop datamap if exists datamap28")
sql("create datamap datamap28 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization),sum(t2.empname) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
val frame = sql(
"select t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where " +
"t1.empname = t2.empname group by t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap28"))
checkAnswer(frame, sql("select t2.designation, sum(t1.utilization) from fact_table4 t1,fact_table5 t2 where " +
"t1.empname = t2.empname group by t2.designation"))
sql(s"drop datamap datamap28")
}
test("test create datamap with join with group by and sub projection with filter") {
sql("drop datamap if exists datamap29")
sql("create datamap datamap29 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization),sum(t2.empname) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
val frame = sql(
"select t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where " +
"t1.empname = t2.empname and t1.empname='shivani' group by t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap29"))
checkAnswer(frame, sql("select t2.designation, sum(t1.utilization) from fact_table4 t1,fact_table5 t2 where " +
"t1.empname = t2.empname and t1.empname='shivani' group by t2.designation"))
sql(s"drop datamap datamap29")
}
ignore("test create datamap with join with group by with filter") {
sql("drop datamap if exists datamap30")
sql("create datamap datamap30 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization),sum(t2.empname) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
"where t1.empname = t2.empname and t2.designation='SA' group by t1.empname, t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap30"))
checkAnswer(frame, sql("select t1.empname, t2.designation, sum(t1.utilization) from fact_table4 t1,fact_table5 t2 " +
"where t1.empname = t2.empname and t2.designation='SA' group by t1.empname, t2.designation"))
sql(s"drop datamap datamap30")
}
ignore("test create datamap with expression on projection") {
sql(s"drop datamap if exists datamap31")
sql("create datamap datamap31 using 'mv' as select empname, designation, utilization, projectcode from fact_table1 ")
val frame = sql(
"select empname, designation, utilization+projectcode from fact_table1")
val analyzed = frame.queryExecution.analyzed
assert(!verifyMVDataMap(analyzed, "datamap31"))
checkAnswer(frame, sql("select empname, designation, utilization+projectcode from fact_table2"))
sql(s"drop datamap datamap31")
}
test("test create datamap with simple and sub group by query and count agg") {
sql(s"drop datamap if exists datamap32")
sql("create datamap datamap32 using 'mv' as select empname, count(utilization) from fact_table1 group by empname")
val frame = sql("select empname,count(utilization) from fact_table1 where empname='shivani' group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap32"))
checkAnswer(frame, sql("select empname,count(utilization) from fact_table2 where empname='shivani' group by empname"))
sql(s"drop datamap datamap32")
}
ignore("test create datamap with simple and sub group by query and avg agg") {
sql(s"drop datamap if exists datamap33")
sql("create datamap datamap33 using 'mv' as select empname, avg(utilization) from fact_table1 group by empname")
val frame = sql("select empname,avg(utilization) from fact_table1 where empname='shivani' group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap33"))
checkAnswer(frame, sql("select empname,avg(utilization) from fact_table2 where empname='shivani' group by empname"))
sql(s"drop datamap datamap33")
}
ignore("test create datamap with left join with group by") {
sql("drop datamap if exists datamap34")
sql("create datamap datamap34 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 " +
"on t1.empname = t2.empname group by t1.empname, t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap34"))
checkAnswer(frame, sql("select t1.empname, t2.designation, sum(t1.utilization) from fact_table4 t1 left join fact_table5 t2 " +
"on t1.empname = t2.empname group by t1.empname, t2.designation"))
sql(s"drop datamap datamap34")
}
ignore("test create datamap with simple and group by query with filter on datamap but not on projection") {
sql("create datamap datamap35 using 'mv' as select designation, sum(utilization) from fact_table1 where empname='shivani' group by designation")
val frame = sql(
"select designation, sum(utilization) from fact_table1 where empname='shivani' group by designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap35"))
checkAnswer(frame, sql("select designation, sum(utilization) from fact_table2 where empname='shivani' group by designation"))
sql(s"drop datamap datamap35")
}
ignore("test create datamap with simple and sub group by query with filter on datamap but not on projection") {
sql("create datamap datamap36 using 'mv' as select designation, sum(utilization) from fact_table1 where empname='shivani' group by designation")
val frame = sql(
"select sum(utilization) from fact_table1 where empname='shivani' group by designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap36"))
checkAnswer(frame, sql("select sum(utilization) from fact_table2 where empname='shivani' group by designation"))
sql(s"drop datamap datamap36")
}
test("test create datamap with agg push join with sub group by ") {
sql("drop datamap if exists datamap37")
sql("create datamap datamap37 using 'mv' as select empname, designation, sum(utilization) from fact_table1 group by empname, designation")
val frame = sql(
"select t1.empname, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
"where t1.empname = t2.empname group by t1.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap37"))
checkAnswer(frame, sql("select t1.empname, sum(t1.utilization) from fact_table3 t1,fact_table4 t2 " +
"where t1.empname = t2.empname group by t1.empname, t1.designation"))
sql(s"drop datamap datamap37")
}
test("test create datamap with agg push join with group by ") {
sql("drop datamap if exists datamap38")
sql("create datamap datamap38 using 'mv' as select empname, designation, sum(utilization) from fact_table1 group by empname, designation")
val frame = sql(
"select t1.empname, t1.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
"where t1.empname = t2.empname group by t1.empname,t1.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap38"))
checkAnswer(frame, sql("select t1.empname,t1.designation, sum(t1.utilization) from fact_table3 t1,fact_table4 t2 " +
"where t1.empname = t2.empname group by t1.empname, t1.designation"))
sql(s"drop datamap datamap38")
}
ignore("test create datamap with agg push join with group by with filter") {
sql("drop datamap if exists datamap39")
sql("create datamap datamap39 using 'mv' as select empname, designation, sum(utilization) from fact_table1 group by empname, designation ")
val frame = sql(
"select t1.empname, t1.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
"where t1.empname = t2.empname and t1.empname='shivani' group by t1.empname,t1.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap39"))
checkAnswer(frame, sql("select t1.empname,t1.designation, sum(t1.utilization) from fact_table3 t1,fact_table4 t2 " +
"where t1.empname = t2.empname and t1.empname='shivani' group by t1.empname, t1.designation"))
sql(s"drop datamap datamap39")
}
test("test create datamap with more agg push join with group by with filter") {
sql("drop datamap if exists datamap40")
sql("create datamap datamap40 using 'mv' as select empname, designation, sum(utilization), count(utilization) from fact_table1 group by empname, designation ")
val frame = sql(
"select t1.empname, t1.designation, sum(t1.utilization),count(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
"where t1.empname = t2.empname and t1.empname='shivani' group by t1.empname,t1.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap40"))
checkAnswer(frame, sql("select t1.empname, t1.designation, sum(t1.utilization),count(t1.utilization) from fact_table3 t1,fact_table4 t2 " +
"where t1.empname = t2.empname and t1.empname='shivani' group by t1.empname,t1.designation"))
sql(s"drop datamap datamap40")
}
ignore("test create datamap with left join with group by with filter") {
sql("drop datamap if exists datamap41")
sql("create datamap datamap41 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 " +
"on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname, t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap41"))
checkAnswer(frame, sql("select t1.empname, t2.designation, sum(t1.utilization) from fact_table4 t1 left join fact_table5 t2 " +
"on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname, t2.designation"))
sql(s"drop datamap datamap41")
}
ignore("test create datamap with left join with sub group by") {
sql("drop datamap if exists datamap42")
sql("create datamap datamap42 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 " +
"on t1.empname = t2.empname group by t1.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap42"))
checkAnswer(frame, sql("select t1.empname, sum(t1.utilization) from fact_table4 t1 left join fact_table5 t2 " +
"on t1.empname = t2.empname group by t1.empname"))
sql(s"drop datamap datamap42")
}
ignore("test create datamap with left join with sub group by with filter") {
sql("drop datamap if exists datamap43")
sql("create datamap datamap43 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 " +
"on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap43"))
checkAnswer(frame, sql("select t1.empname, sum(t1.utilization) from fact_table4 t1 left join fact_table5 t2 " +
"on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname"))
sql(s"drop datamap datamap43")
}
ignore("test create datamap with left join with sub group by with filter on mv") {
sql("drop datamap if exists datamap44")
sql("create datamap datamap44 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname, t2.designation")
val frame = sql(
"select t1.empname, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 " +
"on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap44"))
checkAnswer(frame, sql("select t1.empname, sum(t1.utilization) from fact_table4 t1 left join fact_table5 t2 " +
"on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname"))
sql(s"drop datamap datamap44")
}
test("test create datamap with left join on query and equi join on mv with group by with filter") {
sql("drop datamap if exists datamap45")
sql("create datamap datamap45 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization),sum(t2.empname) from fact_table1 t1 join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
// During spark optimizer it converts the left outer join queries with equi join if any filter present on right side table
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 " +
"on t1.empname = t2.empname where t2.designation='SA' group by t1.empname, t2.designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap45"))
checkAnswer(frame, sql("select t1.empname, t2.designation, sum(t1.utilization) from fact_table4 t1 left join fact_table5 t2 " +
"on t1.empname = t2.empname where t2.designation='SA' group by t1.empname, t2.designation"))
sql(s"drop datamap datamap45")
}
test("jira carbondata-2523") {
sql("drop datamap if exists mv13")
sql("drop table if exists test4")
sql("create table test4 ( name string,age int,salary int) stored by 'carbondata'")
sql(" insert into test4 select 'babu',12,12").show()
sql("create datamap mv13 using 'mv' as select name,sum(salary) from test4 group by name")
val frame = sql(
"select name,sum(salary) from test4 group by name")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "mv13"))
}
test("jira carbondata-2528-1") {
sql("drop datamap if exists MV_order")
sql("create datamap MV_order using 'mv' as select empname,sum(salary) as total from fact_table1 group by empname")
val frame = sql(
"select empname,sum(salary) as total from fact_table1 group by empname order by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_order"))
}
test("jira carbondata-2528-2") {
sql("drop datamap if exists MV_order")
sql("drop datamap if exists MV_desc_order")
sql("create datamap MV_order using 'mv' as select empname,sum(salary)+sum(utilization) as total from fact_table1 group by empname")
val frame = sql(
"select empname,sum(salary)+sum(utilization) as total from fact_table1 group by empname order by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_order"))
}
test("jira carbondata-2528-3") {
sql("drop datamap if exists MV_order")
sql("create datamap MV_order using 'mv' as select empname,sum(salary)+sum(utilization) as total from fact_table1 group by empname order by empname DESC")
val frame = sql(
"select empname,sum(salary)+sum(utilization) as total from fact_table1 group by empname order by empname DESC")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_order"))
sql("drop datamap if exists MV_order")
}
test("jira carbondata-2528-4") {
sql("drop datamap if exists MV_order")
sql("create datamap MV_order using 'mv' as select empname,sum(salary)+sum(utilization) as total from fact_table1 group by empname order by empname DESC")
val frame = sql(
"select empname,sum(salary)+sum(utilization) as total from fact_table1 where empname = 'ravi' group by empname order by empname DESC")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_order"))
sql("drop datamap if exists MV_order")
}
test("jira carbondata-2530") {
sql("drop table if exists test1")
sql("drop datamap if exists datamv2")
sql("create table test1( name string,country string,age int,salary int) stored by 'carbondata'")
sql("insert into test1 select 'name1','USA',12,23")
sql("create datamap datamv2 using 'mv' as select country,sum(salary) from test1 group by country")
val frame = sql("select country,sum(salary) from test1 where country='USA' group by country")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamv2"))
sql("insert into test1 select 'name1','USA',12,23")
val frame1 = sql("select country,sum(salary) from test1 where country='USA' group by country")
val analyzed1 = frame1.queryExecution.analyzed
assert(verifyMVDataMap(analyzed1, "datamv2"))
sql("drop datamap if exists datamv2")
sql("drop table if exists test1")
}
test("jira carbondata-2534") {
sql("drop datamap if exists MV_exp")
sql("create datamap MV_exp using 'mv' as select sum(salary),substring(empname,2,5),designation from fact_table1 group by substring(empname,2,5),designation")
val frame = sql(
"select sum(salary),substring(empname,2,5),designation from fact_table1 group by substring(empname,2,5),designation")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_exp"))
sql("drop datamap if exists MV_exp")
}
test("jira carbondata-2542") {
sql("""drop database if exists xy cascade""")
sql("""create database if not exists xy""")
sql(
"""
| CREATE TABLE xy.fact_tablexy (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql("drop datamap if exists MV_exp")
sql("create datamap MV_exp using 'mv' as select doj,sum(salary) from xy.fact_tablexy group by doj")
val frame = sql(
"select doj,sum(salary) from xy.fact_tablexy group by doj")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_exp"))
sql("drop datamap if exists MV_exp")
sql("""drop database if exists xy cascade""")
}
test("jira carbondata-2550") {
sql("drop table if exists mvtable1")
sql("drop datamap if exists map1")
sql("create table mvtable1(name string,age int,salary int) stored by 'carbondata'")
sql(" insert into mvtable1 select 'n1',12,12")
sql(" insert into mvtable1 select 'n1',12,12")
sql(" insert into mvtable1 select 'n3',12,12")
sql(" insert into mvtable1 select 'n4',12,12")
sql("create datamap map1 using 'mv' as select name,sum(salary) from mvtable1 group by name")
val frame = sql("select name,sum(salary) from mvtable1 group by name limit 1")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "map1"))
sql("drop datamap if exists map1")
sql("drop table if exists mvtable1")
}
test("jira carbondata-2576") {
sql("drop datamap if exists datamap_comp_maxsumminavg")
sql("create datamap datamap_comp_maxsumminavg using 'mv' as select empname,max(projectenddate),sum(salary),min(projectjoindate),avg(attendance) from fact_table1 group by empname")
val frame = sql(
"select empname,max(projectenddate),sum(salary),min(projectjoindate),avg(attendance) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap_comp_maxsumminavg"))
sql("drop datamap if exists datamap_comp_maxsumminavg")
}
test("jira carbondata-2540") {
sql("drop datamap if exists mv_unional")
intercept[UnsupportedOperationException] {
sql(
"create datamap mv_unional using 'mv' as Select Z.deptname From (Select deptname,empname From fact_table1 Union All Select deptname,empname from fact_table2) Z")
}
sql("drop datamap if exists mv_unional")
}
test("jira carbondata-2533") {
sql("drop datamap if exists MV_exp")
intercept[UnsupportedOperationException] {
sql(
"create datamap MV_exp using 'mv' as select sum(case when deptno=11 and (utilization=92) then salary else 0 end) as t from fact_table1 group by empname")
val frame = sql(
"select sum(case when deptno=11 and (utilization=92) then salary else 0 end) as t from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_exp"))
}
sql("drop datamap if exists MV_exp")
}
test("jira carbondata-2560") {
sql("drop datamap if exists MV_exp1")
sql("drop datamap if exists MV_exp2")
sql("create datamap MV_exp1 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
intercept[UnsupportedOperationException] {
sql(
"create datamap MV_exp2 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
}
sql("show datamap").show()
val frame = sql(
"select empname, sum(utilization) from fact_table1 group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "MV_exp1"))
sql("drop datamap if exists MV_exp1")
sql("drop datamap if exists MV_exp2")
}
test("jira carbondata-2531") {
sql("drop datamap if exists datamap46")
sql("create datamap datamap46 using 'mv' as select deptname, sum(salary) from fact_table1 group by deptname")
val frame = sql(
"select deptname as babu, sum(salary) from fact_table1 as tt group by deptname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap46"))
sql("drop datamap if exists datamap46")
}
test("jira carbondata-2539") {
sql("drop datamap if exists datamap_subqry")
sql("create datamap datamap_subqry using 'mv' as select empname, min(salary) from fact_table1 group by empname")
val frame = sql(
"SELECT max(utilization) FROM fact_table1 WHERE salary IN (select min(salary) from fact_table1 group by empname ) group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap_subqry"))
sql("drop datamap if exists datamap_subqry")
}
test("jira carbondata-2539-1") {
sql("drop datamap if exists datamap_comp_maxsumminavg")
sql("create datamap datamap_comp_maxsumminavg using 'mv' as select empname,max(projectenddate),sum(salary),min(projectjoindate),avg(attendance) from fact_table1 group by empname")
sql("drop datamap if exists datamap_subqry")
sql("create datamap datamap_subqry using 'mv' as select min(salary) from fact_table1")
val frame = sql(
"SELECT max(utilization) FROM fact_table1 WHERE salary IN (select min(salary) from fact_table1) group by empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap_subqry"))
sql("drop datamap if exists datamap_subqry")
}
test("basic scenario") {
sql("drop table if exists mvtable1")
sql("drop table if exists mvtable2")
sql("create table mvtable1(name string,age int,salary int) stored by 'carbondata'")
sql("create table mvtable2(name string,age int,salary int) stored by 'carbondata'")
sql("create datamap MV11 using 'mv' as select name from mvtable2")
sql(" insert into mvtable1 select 'n1',12,12")
sql(" insert into mvtable1 select 'n1',12,12")
sql(" insert into mvtable1 select 'n3',12,12")
sql(" insert into mvtable1 select 'n4',12,12")
sql("update mvtable1 set(name) = ('updatedName')").show()
checkAnswer(sql("select count(*) from mvtable1 where name = 'updatedName'"),Seq(Row(4)))
sql("drop table if exists mvtable1")
sql("drop table if exists mvtable2")
}
test("test create datamap with streaming table") {
sql("drop datamap if exists dm_stream_test1")
sql("drop datamap if exists dm_stream_bloom")
sql("drop datamap if exists dm_stream_PreAggMax")
sql("drop table if exists fact_streaming_table1")
sql(
"""
| CREATE TABLE fact_streaming_table1 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
| tblproperties('streaming'='true')
""".stripMargin)
sql(
s"""
| CREATE DATAMAP dm_stream_bloom ON TABLE fact_streaming_table1
| USING 'bloomfilter'
| DMProperties('INDEX_COLUMNS'='empname,deptname', 'BLOOM_SIZE'='640000')
""".stripMargin)
sql("create datamap dm_stream_PreAggMax on table fact_streaming_table1 using 'preaggregate' " +
"as select empname,max(salary) as max from fact_streaming_table1 group by empname")
val exception_tb_mv: Exception = intercept[Exception] {
sql("create datamap dm_stream_test1 using 'mv' as select empname, sum(utilization) from " +
"fact_streaming_table1 group by empname")
}
assert(exception_tb_mv.getMessage
.contains("Streaming table does not support creating MV datamap"))
}
test("test create datamap with streaming table join carbon table and join non-carbon table ") {
sql("drop datamap if exists dm_stream_test2")
sql("drop table if exists fact_streaming_table2")
sql("drop table if exists fact_table_parquet")
sql(
"""
| CREATE TABLE fact_streaming_table2 (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED BY 'org.apache.carbondata.format'
| tblproperties('streaming'='true')
""".stripMargin)
sql(
"""
| CREATE TABLE fact_table_parquet (empname String, designation String, doj Timestamp,
| workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| STORED AS parquet
""".stripMargin)
val exception_tb_mv2: Exception = intercept[Exception] {
sql("create datamap dm_stream_test2 using 'mv' as select t1.empname as c1, t2.designation, " +
"t2.empname as c2,t3.empname from (fact_table1 t1 inner join fact_streaming_table2 t2 " +
"on (t1.empname = t2.empname)) inner join fact_table_parquet t3 " +
"on (t1.empname = t3.empname)")
}
assert(exception_tb_mv2.getMessage
.contains("Streaming table does not support creating MV datamap"))
}
test("test set streaming property of the table which has MV datamap") {
sql("drop datamap if exists dm_stream_test3")
sql("create datamap dm_stream_test3 using 'mv' as select empname, sum(utilization) from " +
"fact_table1 group by empname")
val exception_tb_mv3: Exception = intercept[Exception] {
sql("alter table fact_table1 set tblproperties('streaming'='true')")
}
assert(exception_tb_mv3.getMessage
.contains("The table which has MV datamap does not support set streaming property"))
sql("drop datamap if exists dm_stream_test3")
}
test("select mv stack exception") {
val querySQL = "select sum(x12) as y1, sum(x13) as y2, sum(x14) as y3,sum(x15) " +
"as y4,X8,x9,x1 from all_table group by X8,x9,x1"
sql("drop datamap if exists all_table_mv")
sql("drop table if exists all_table")
sql("""
| create table all_table(x1 bigint,x2 bigint,
| x3 string,x4 bigint,x5 bigint,x6 int,x7 string,x8 int, x9 int,x10 bigint,
| x11 bigint, x12 bigint,x13 bigint,x14 bigint,x15 bigint,x16 bigint,
| x17 bigint,x18 bigint,x19 bigint) stored by 'carbondata'""".stripMargin)
sql("insert into all_table select 1,1,null,1,1,1,null,1,1,1,1,1,1,1,1,1,1,1,1")
sql("create datamap all_table_mv on table all_table using 'mv' as " + querySQL)
val frame = sql(querySQL)
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "all_table_mv"))
assert(1 == frame.collect().size)
sql("drop table if exists all_table")
}
test("test select * and distinct when MV is enabled") {
sql("drop table if exists limit_fail")
sql("CREATE TABLE limit_fail (empname String, designation String, doj Timestamp,workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int)STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA local inpath '$resourcesPath/data_big.csv' INTO TABLE limit_fail OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')")
sql("create datamap limit_fail_dm1 using 'mv' as select empname,designation from limit_fail")
try {
val df = sql("select distinct(empname) from limit_fail limit 10")
sql("select * from limit_fail limit 10").show()
val analyzed = df.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "limit_fail_dm1"))
} catch {
case ex: Exception =>
assert(false)
}
}
test(" test MV with like queries and filter queries") {
sql("drop table if exists mv_like")
sql("create table mv_like(name string, age int, address string, Country string, id int) stored by 'carbondata'")
sql("create datamap mvlikedm1 using 'mv' as select name,address from mv_like where Country NOT LIKE 'US' group by name,address")
sql("create datamap mvlikedm2 using 'mv' as select name,address,Country from mv_like where Country = 'US' or Country = 'China' group by name,address,Country")
sql("insert into mv_like select 'chandler', 32, 'newYork', 'US', 5")
val df1 = sql("select name,address from mv_like where Country NOT LIKE 'US' group by name,address")
val analyzed1 = df1.queryExecution.analyzed
assert(verifyMVDataMap(analyzed1, "mvlikedm1"))
val df2 = sql("select name,address,Country from mv_like where Country = 'US' or Country = 'China' group by name,address,Country")
val analyzed2 = df2.queryExecution.analyzed
assert(verifyMVDataMap(analyzed2, "mvlikedm2"))
}
def verifyMVDataMap(logicalPlan: LogicalPlan, dataMapName: String): Boolean = {
val tables = logicalPlan collect {
case l: LogicalRelation => l.catalogTable.get
}
tables.exists(_.identifier.table.equalsIgnoreCase(dataMapName+"_table"))
}
def drop(): Unit = {
sql("drop table IF EXISTS fact_table1")
sql("drop table IF EXISTS fact_table2")
sql("drop table IF EXISTS fact_table3")
sql("drop table IF EXISTS fact_table4")
sql("drop table IF EXISTS fact_table5")
sql("drop table IF EXISTS fact_table6")