Skip to content

Commit

Permalink
[SPARK-20535][SPARKR] R wrappers for explode_outer and posexplode_outer
Browse files Browse the repository at this point in the history
## What changes were proposed in this pull request?

Ad R wrappers for

- `o.a.s.sql.functions.explode_outer`
- `o.a.s.sql.functions.posexplode_outer`

## How was this patch tested?

Additional unit tests, manual testing.

Author: zero323 <zero323@users.noreply.github.com>

Closes #17809 from zero323/SPARK-20535.
  • Loading branch information
zero323 authored and Felix Cheung committed Apr 30, 2017
1 parent 1ee494d commit ae3df4e
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 0 deletions.
2 changes: 2 additions & 0 deletions R/pkg/NAMESPACE
Expand Up @@ -234,6 +234,7 @@ exportMethods("%in%",
"endsWith",
"exp",
"explode",
"explode_outer",
"expm1",
"expr",
"factorial",
Expand Down Expand Up @@ -296,6 +297,7 @@ exportMethods("%in%",
"percent_rank",
"pmod",
"posexplode",
"posexplode_outer",
"quarter",
"rand",
"randn",
Expand Down
56 changes: 56 additions & 0 deletions R/pkg/R/functions.R
Expand Up @@ -3803,3 +3803,59 @@ setMethod("repeat_string",
jc <- callJStatic("org.apache.spark.sql.functions", "repeat", x@jc, numToInt(n))
column(jc)
})

#' explode_outer
#'
#' Creates a new row for each element in the given array or map column.
#' Unlike \code{explode}, if the array/map is \code{null} or empty
#' then \code{null} is produced.
#'
#' @param x Column to compute on
#'
#' @rdname explode_outer
#' @name explode_outer
#' @family collection_funcs
#' @aliases explode_outer,Column-method
#' @export
#' @examples \dontrun{
#' df <- createDataFrame(data.frame(
#' id = c(1, 2, 3), text = c("a,b,c", NA, "d,e")
#' ))
#'
#' head(select(df, df$id, explode_outer(split_string(df$text, ","))))
#' }
#' @note explode_outer since 2.3.0
setMethod("explode_outer",
signature(x = "Column"),
function(x) {
jc <- callJStatic("org.apache.spark.sql.functions", "explode_outer", x@jc)
column(jc)
})

#' posexplode_outer
#'
#' Creates a new row for each element with position in the given array or map column.
#' Unlike \code{posexplode}, if the array/map is \code{null} or empty
#' then the row (\code{null}, \code{null}) is produced.
#'
#' @param x Column to compute on
#'
#' @rdname posexplode_outer
#' @name posexplode_outer
#' @family collection_funcs
#' @aliases posexplode_outer,Column-method
#' @export
#' @examples \dontrun{
#' df <- createDataFrame(data.frame(
#' id = c(1, 2, 3), text = c("a,b,c", NA, "d,e")
#' ))
#'
#' head(select(df, df$id, posexplode_outer(split_string(df$text, ","))))
#' }
#' @note posexplode_outer since 2.3.0
setMethod("posexplode_outer",
signature(x = "Column"),
function(x) {
jc <- callJStatic("org.apache.spark.sql.functions", "posexplode_outer", x@jc)
column(jc)
})
8 changes: 8 additions & 0 deletions R/pkg/R/generics.R
Expand Up @@ -1016,6 +1016,10 @@ setGeneric("encode", function(x, charset) { standardGeneric("encode") })
#' @export
setGeneric("explode", function(x) { standardGeneric("explode") })

#' @rdname explode_outer
#' @export
setGeneric("explode_outer", function(x) { standardGeneric("explode_outer") })

#' @rdname expr
#' @export
setGeneric("expr", function(x) { standardGeneric("expr") })
Expand Down Expand Up @@ -1175,6 +1179,10 @@ setGeneric("pmod", function(y, x) { standardGeneric("pmod") })
#' @export
setGeneric("posexplode", function(x) { standardGeneric("posexplode") })

#' @rdname posexplode_outer
#' @export
setGeneric("posexplode_outer", function(x) { standardGeneric("posexplode_outer") })

#' @rdname quarter
#' @export
setGeneric("quarter", function(x) { standardGeneric("quarter") })
Expand Down
1 change: 1 addition & 0 deletions R/pkg/inst/tests/testthat/test_sparkSQL.R
Expand Up @@ -1347,6 +1347,7 @@ test_that("column functions", {
c18 <- covar_pop(c, c1) + covar_pop("c", "c1")
c19 <- spark_partition_id() + coalesce(c) + coalesce(c1, c2, c3)
c20 <- to_timestamp(c) + to_timestamp(c, "yyyy") + to_date(c, "yyyy")
c21 <- posexplode_outer(c) + explode_outer(c)

# Test if base::is.nan() is exposed
expect_equal(is.nan(c("a", "b")), c(FALSE, FALSE))
Expand Down

0 comments on commit ae3df4e

Please sign in to comment.