diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala index ced6fef72d45..c6eccac2bc50 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala @@ -901,4 +901,47 @@ class TestInsertTable extends HoodieSparkSqlTestBase { } } } + + test("Test enable hoodie.merge.allow.duplicate.on.inserts when write") { + spark.sql("set hoodie.datasource.write.operation = insert") + Seq("mor", "cow").foreach { tableType => + withTempDir { tmp => + val tableName = generateTableName + spark.sql( + s""" + | create table $tableName ( + | id int, + | name string, + | price double, + | ts long, + | dt string + | ) using hudi + | partitioned by (dt) + | location '${tmp.getCanonicalPath}/$tableName' + | tblproperties ( + | primaryKey = 'id', + | preCombineField = 'ts', + | type = '$tableType' + | ) + """.stripMargin) + spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a1', 10, 1000)") + checkAnswer(s"select id, name, price, ts, dt from $tableName")( + Seq(1, "a1", 10, 1000, "2021-12-25") + ) + spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false") + spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a2', 20, 1001)") + checkAnswer(s"select id, name, price, ts, dt from $tableName")( + Seq(1, "a2", 20, 1001, "2021-12-25") + ) + spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true") + spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a3', 30, 1002)") + checkAnswer(s"select id, name, price, ts, dt from $tableName")( + Seq(1, "a2", 20, 1001, "2021-12-25"), + Seq(1, "a3", 30, 1002, "2021-12-25") + ) + } + } + spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false") + spark.sql("set hoodie.datasource.write.operation = upsert") + } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala index 8a6aa9691d93..c0c996e5415b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala @@ -609,7 +609,6 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase { | preCombineField = 'ts' | ) """.stripMargin) - spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true") // Insert data spark.sql(s"insert into $tableName select 1, 1, 'a1', 1, 10, '2022-08-18'") checkAnswer(s"select id1, id2, name, price, ts, dt from $tableName")(