Skip to content

Commit

Permalink
Remove the workaroud to allow dots in field names in R's createDataFame
Browse files Browse the repository at this point in the history
  • Loading branch information
HyukjinKwon committed Jul 26, 2017
1 parent 481f079 commit ef9d200
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 10 deletions.
10 changes: 0 additions & 10 deletions R/pkg/R/SQLContext.R
Expand Up @@ -262,16 +262,6 @@ createDataFrame.default <- function(data, schema = NULL, samplingRatio = 1.0,
})
}

# SPAKR-SQL does not support '.' in column name, so replace it with '_'
# TODO(davies): remove this once SPARK-2775 is fixed
names <- lapply(names, function(n) {
nn <- gsub("[.]", "_", n)
if (nn != n) {
warning(paste("Use", nn, "instead of", n, " as column name"))
}
nn
})

types <- lapply(row, infer_type)
fields <- lapply(1:length(row), function(i) {
structField(names[[i]], types[[i]], TRUE)
Expand Down
6 changes: 6 additions & 0 deletions R/pkg/tests/fulltests/test_sparkSQL.R
Expand Up @@ -462,10 +462,16 @@ test_that("create DataFrame from list or data.frame", {
df <- createDataFrame(l, c("a", "b"))
expect_equal(columns(df), c("a", "b"))

df <- createDataFrame(list(list(1, 2)), list("..a", "..."))
expect_equal(columns(df), c("..a", "..."))

l <- list(list(a = 1, b = 2), list(a = 3, b = 4))
df <- createDataFrame(l)
expect_equal(columns(df), c("a", "b"))

df <- createDataFrame(list(list(a.a = 1, a.. = 2)))
expect_equal(columns(df), c("a.a", "a.."))

a <- 1:3
b <- c("a", "b", "c")
ldf <- data.frame(a, b)
Expand Down

0 comments on commit ef9d200

Please sign in to comment.