remove embedded nul bytes if present

Signed-off-by: Jozef <jozef.hajnala@gmail.com>
sparklyr · Feb 2, 2020 · 8c86497 · 8c86497
1 parent 12b1d87
commit 8c86497
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 0 deletions.
diff --git a/R/core_deserialize.R b/R/core_deserialize.R
@@ -85,6 +85,10 @@ readString <- function(con) {
 
   if (stringLen > 0) {
     raw <- read_bin(con, raw(), stringLen, endian = "big")
+    if (is.element("00", raw)) {
+     warning("Input contains embedded nuls, removing.")
+     raw <- raw[raw != "00"]
+    }
     string <- rawToChar(raw)
   }
 

diff --git a/tests/testthat/embeddednul/with_embedded_nul.csv b/tests/testthat/embeddednul/with_embedded_nul.csv
diff --git a/tests/testthat/test-read-write.R b/tests/testthat/test-read-write.R
@@ -239,5 +239,17 @@ test_that("spark_read_csv() can read verbatim column types", {
       c = list(name = "c", type = "DoubleType")
     )
   )
+})
 
+test_that("spark_read_csv() can read if embedded nuls present", {
+  fpath <- dir(getwd(), "with_embedded_nul\\.csv$", full.names = TRUE, recursive = TRUE)
+  df <- spark_read_csv(sc, name = "test_embedded_nul", path = fpath)
+  expect_equal(
+    suppressWarnings(df %>% collect()),
+    data.frame(test = "teststring", stringsAsFactors = FALSE)
+  )
+  expect_warning(
+    df %>% collect(),
+    "Input contains embedded nuls, removing."
+  )
 })