Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-9324][SPARK-9322][SPARK-9321][SPARKR] Some aliases for R-like functions in DataFrames #7764

Closed
wants to merge 6 commits into from
Closed

[SPARK-9324][SPARK-9322][SPARK-9321][SPARKR] Some aliases for R-like functions in DataFrames #7764

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b5aa988
Added dim, ncol, nrow, names, rbind, and unique functions to DataFrames
falaki Jul 30, 2015
d9307f8
Added tests
falaki Jul 30, 2015
c1b88bd
Moved setGeneric and other comments applied
falaki Jul 30, 2015
f51cbef
Merge branch 'master' into sparkR-alias
falaki Jul 30, 2015
5e4a4d0
Removed extra code
falaki Jul 30, 2015
56016f5
Updated R documentation
falaki Jul 31, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions R/pkg/NAMESPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ exportMethods("arrange",
"count",
"crosstab",
"describe",
"dim",
"distinct",
"dropna",
"dtypes",
Expand All @@ -44,11 +45,15 @@ exportMethods("arrange",
"isLocal",
"join",
"limit",
"names",
"ncol",
"nrow",
"orderBy",
"mutate",
"names",
"persist",
"printSchema",
"rbind",
"registerTempTable",
"rename",
"repartition",
Expand All @@ -65,6 +70,7 @@ exportMethods("arrange",
"summarize",
"take",
"unionAll",
"unique",
"unpersist",
"where",
"withColumn",
Expand Down
83 changes: 83 additions & 0 deletions R/pkg/R/DataFrame.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,16 @@ setMethod("names",
columns(x)
})

#' @rdname columns
setMethod("names<-",
signature(x = "DataFrame"),
function(x, value) {
if (!is.null(value)) {
sdf <- callJMethod(x@sdf, "toDF", listToSeq(as.list(value)))
dataFrame(sdf)
}
})

#' Register Temporary Table
#'
#' Registers a DataFrame as a Temporary Table in the SQLContext
Expand Down Expand Up @@ -473,6 +483,14 @@ setMethod("distinct",
dataFrame(sdf)
})

#' @rdname unique
#' @aliases unique
setMethod("unique",
signature(x = "DataFrame"),
function(x) {
distinct(x)
})

#' Sample
#'
#' Return a sampled subset of this DataFrame using a random seed.
Expand Down Expand Up @@ -534,6 +552,53 @@ setMethod("count",
callJMethod(x@sdf, "count")
})

#' @rdname nrow
#' @aliases count
setMethod("nrow",
signature(x = "DataFrame"),
function(x) {
count(x)
})

#' Returns the number of columns in a DataFrame
#'
#' @param x a SparkSQL DataFrame
#'
#' @rdname ncol
#' @export
#' @examples
#'\dontrun{
#' sc <- sparkR.init()
#' sqlContext <- sparkRSQL.init(sc)
#' path <- "path/to/file.json"
#' df <- jsonFile(sqlContext, path)
#' ncol(df)
#' }
setMethod("ncol",
signature(x = "DataFrame"),
function(x) {
length(columns(x))
})

#' Returns the dimentions (number for rows and columns) of a DataFrame
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo: number of

#' @param x a SparkSQL DataFrame
#'
#' @rdname dim
#' @export
#' @examples
#'\dontrun{
#' sc <- sparkR.init()
#' sqlContext <- sparkRSQL.init(sc)
#' path <- "path/to/file.json"
#' df <- jsonFile(sqlContext, path)
#' dim(df)
#' }
setMethod("dim",
signature(x = "DataFrame"),
function(x) {
c(count(x), ncol(x))
})

#' Collects all the elements of a Spark DataFrame and coerces them into an R data.frame.
#'
#' @param x A SparkSQL DataFrame
Expand Down Expand Up @@ -1231,6 +1296,24 @@ setMethod("unionAll",
dataFrame(unioned)
})

setGeneric("rbind", signature = "...")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets move this to generics.R ?


rbind.SparkDataFrames <- function(x, ..., deparse.level = 1) {
allargs <- list(...)
if (nargs() == 3) {
unionAll(x, ...)
} else {
unionAll(x, Recall(..., deparse.level = 1))
}
}

#' @rdname rbind
#' @aliases unionAll
setMethod("rbind",
signature(... = "DataFrame"),
rbind.SparkDataFrames
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any reason to not just inline the rbind.SparkDataFrames function here ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure.
BTW: what do you think about letting unionAll accept more than two DataFrames?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Its a good feature to have but I think its also fine to just support the two argument case if the other one adds too much complexity

)

#' Intersect
#'
#' Return a new DataFrame containing rows only in both this DataFrame
Expand Down
22 changes: 19 additions & 3 deletions R/pkg/inst/tests/test_sparkSQL.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ test_that("create DataFrame from RDD", {
df <- createDataFrame(sqlContext, rdd, list("a", "b"))
expect_is(df, "DataFrame")
expect_equal(count(df), 10)
expect_equal(nrow(df), 10)
expect_equal(ncol(df), 2)
expect_equal(dim(df), c(10, 2))
expect_equal(columns(df), c("a", "b"))
expect_equal(dtypes(df), list(c("a", "int"), c("b", "string")))

Expand Down Expand Up @@ -488,7 +491,7 @@ test_that("head() and first() return the correct data", {
expect_equal(nrow(testFirst), 1)
})

test_that("distinct() on DataFrames", {
test_that("distinct() and unique on DataFrames", {
lines <- c("{\"name\":\"Michael\"}",
"{\"name\":\"Andy\", \"age\":30}",
"{\"name\":\"Justin\", \"age\":19}",
Expand All @@ -500,6 +503,10 @@ test_that("distinct() on DataFrames", {
uniques <- distinct(df)
expect_is(uniques, "DataFrame")
expect_equal(count(uniques), 3)

uniques2 <- unique(df)
expect_is(uniques2, "DataFrame")
expect_equal(count(uniques2), 3)
})

test_that("sample on a DataFrame", {
Expand Down Expand Up @@ -809,7 +816,7 @@ test_that("isLocal()", {
expect_false(isLocal(df))
})

test_that("unionAll(), except(), and intersect() on a DataFrame", {
test_that("unionAll(), rbind(), except(), and intersect() on a DataFrame", {
df <- jsonFile(sqlContext, jsonPath)

lines <- c("{\"name\":\"Bob\", \"age\":24}",
Expand All @@ -824,6 +831,11 @@ test_that("unionAll(), except(), and intersect() on a DataFrame", {
expect_equal(count(unioned), 6)
expect_equal(first(unioned)$name, "Michael")

unioned2 <- arrange(rbind(unioned, df, df2), df$age)
expect_is(unioned2, "DataFrame")
expect_equal(count(unioned2), 12)
expect_equal(first(unioned2)$name, "Michael")

excepted <- arrange(except(df, df2), desc(df$age))
expect_is(unioned, "DataFrame")
expect_equal(count(excepted), 2)
Expand All @@ -847,7 +859,7 @@ test_that("withColumn() and withColumnRenamed()", {
expect_equal(columns(newDF2)[1], "newerAge")
})

test_that("mutate() and rename()", {
test_that("mutate(), rename() and names()", {
df <- jsonFile(sqlContext, jsonPath)
newDF <- mutate(df, newAge = df$age + 2)
expect_equal(length(columns(newDF)), 3)
Expand All @@ -857,6 +869,10 @@ test_that("mutate() and rename()", {
newDF2 <- rename(df, newerAge = df$age)
expect_equal(length(columns(newDF2)), 2)
expect_equal(columns(newDF2)[1], "newerAge")

names(newDF2) <- c("newerName", "evenNewerAge")
expect_equal(length(names(newDF2)), 2)
expect_equal(names(newDF2)[1], "newerName")
})

test_that("write.df() on DataFrame and works with parquetFile", {
Expand Down