[MINOR] Test case for hoodie.merge.allow.duplicate.on.inserts (apache…

…#6949)
onehouseinc · Dec 14, 2022 · 3cb7fe6 · 3cb7fe6
1 parent 39d9b44
commit 3cb7fe6
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 1 deletion.
diff --git a/...park-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala b/...park-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala
@@ -901,4 +901,47 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
       }
     }
   }
+
+  test("Test enable hoodie.merge.allow.duplicate.on.inserts when write") {
+    spark.sql("set hoodie.datasource.write.operation = insert")
+    Seq("mor", "cow").foreach { tableType =>
+      withTempDir { tmp =>
+        val tableName = generateTableName
+        spark.sql(
+          s"""
+             | create table $tableName (
+             |  id int,
+             |  name string,
+             |  price double,
+             |  ts long,
+             |  dt string
+             | ) using hudi
+             | partitioned by (dt)
+             | location '${tmp.getCanonicalPath}/$tableName'
+             | tblproperties (
+             |  primaryKey = 'id',
+             |  preCombineField = 'ts',
+             |  type = '$tableType'
+             | )
+        """.stripMargin)
+        spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a1', 10, 1000)")
+        checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+          Seq(1, "a1", 10, 1000, "2021-12-25")
+        )
+        spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
+        spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a2', 20, 1001)")
+        checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+          Seq(1, "a2", 20, 1001, "2021-12-25")
+        )
+        spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true")
+        spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a3', 30, 1002)")
+        checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+          Seq(1, "a2", 20, 1001, "2021-12-25"),
+          Seq(1, "a3", 30, 1002, "2021-12-25")
+        )
+      }
+    }
+    spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
+    spark.sql("set hoodie.datasource.write.operation = upsert")
+  }
 }
diff --git a/...-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala b/...-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala
@@ -609,7 +609,6 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
            |  preCombineField = 'ts'
            | )
        """.stripMargin)
-      spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true")
       // Insert data
       spark.sql(s"insert into $tableName select 1, 1, 'a1', 1, 10, '2022-08-18'")
       checkAnswer(s"select id1, id2, name, price, ts, dt from $tableName")(