Updated to ignore missing and corrupt files

databrickslabs · May 24, 2023 · 27bd16c · 27bd16c
1 parent e0da30d
commit 27bd16c
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/main/scala/com/databricks/labs/deltaoms/common/OMSOperations.scala b/src/main/scala/com/databricks/labs/deltaoms/common/OMSOperations.scala
@@ -253,6 +253,8 @@ trait OMSOperations extends Serializable with SparkSettings with Logging with Sc
         .option("cloudFiles.format", "json")
         .option("cloudFiles.maxFilesPerTrigger", maxFilesPerTrigger)
         .option("cloudFiles.useIncrementalListing", "true")
+        .option("ignoreCorruptFiles", "true")
+        .option("ignoreMissingFiles", "true")
         .schema(actionSchema)
         .load(path).select("*", "_metadata"))
     } else {