Merge pull request #4 from apache/master

Pulling functionality from apache spark

.rat-excludes

@@ -86,4 +86,12 @@ local-1430917381535_2
 DESCRIPTION
 NAMESPACE
 test_support/*
+.*Rd
+help/*
+html/*
+INDEX
 .lintr
+gen-java.*
+.*avpr
+org.apache.spark.sql.sources.DataSourceRegister
+.*parquet

LICENSE

@@ -948,6 +948,6 @@ The following components are provided under the MIT License. See project link fo
      (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.5 - http://www.slf4j.org)
      (MIT License) pyrolite (org.spark-project:pyrolite:2.0.1 - http://pythonhosted.org/Pyro4/)
      (MIT License) scopt (com.github.scopt:scopt_2.10:3.2.0 - https://github.com/scopt/scopt)
-     (The MIT License) Mockito (org.mockito:mockito-all:1.8.5 - http://www.mockito.org)
+     (The MIT License) Mockito (org.mockito:mockito-core:1.9.5 - http://www.mockito.org)
      (MIT License) jquery (https://jquery.org/license/)
      (MIT License) AnchorJS (https://github.com/bryanbraun/anchorjs)

R/README.md

@@ -6,7 +6,7 @@ SparkR is an R package that provides a light-weight frontend to use Spark from R
 
 #### Build Spark
 
-Build Spark with [Maven](http://spark.apache.org/docs/latest/building-spark.html#building-with-buildmvn) and include the `-PsparkR` profile to build the R package. For example to use the default Hadoop versions you can run
+Build Spark with [Maven](http://spark.apache.org/docs/latest/building-spark.html#building-with-buildmvn) and include the `-Psparkr` profile to build the R package. For example to use the default Hadoop versions you can run
 ```
   build/mvn -DskipTests -Psparkr package
 ```

R/create-docs.sh

@@ -39,7 +39,7 @@ pushd $FWDIR
 mkdir -p pkg/html
 pushd pkg/html
 
-Rscript -e 'library(SparkR, lib.loc="../../lib"); library(knitr); knit_rd("SparkR")'
+Rscript -e 'libDir <- "../../lib"; library(SparkR, lib.loc=libDir); library(knitr); knit_rd("SparkR", links = tools::findHTMLlinks(paste(libDir, "SparkR", sep="/")))'
 
 popd
 

R/install-dev.sh

@@ -34,12 +34,12 @@ LIB_DIR="$FWDIR/lib"
 
 mkdir -p $LIB_DIR
 
-pushd $FWDIR
+pushd $FWDIR > /dev/null
 
 # Generate Rd files if devtools is installed
 Rscript -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); devtools::document(pkg="./pkg", roclets=c("rd")) }'
 
 # Install SparkR to $LIB_DIR
 R CMD INSTALL --library=$LIB_DIR $FWDIR/pkg/
 
-popd
+popd > /dev/null

R/pkg/.lintr

@@ -1,2 +1,2 @@
-linters: with_defaults(line_length_linter(100), camel_case_linter = NULL)
+linters: with_defaults(line_length_linter(100), camel_case_linter = NULL, open_curly_linter(allow_single_line = TRUE), closed_curly_linter(allow_single_line = TRUE))
 exclusions: list("inst/profile/general.R" = 1, "inst/profile/shell.R")

R/pkg/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: SparkR
 Type: Package
 Title: R frontend for Spark
-Version: 1.4.0
+Version: 1.6.0
 Date: 2013-09-09
 Author: The Apache Software Foundation
 Maintainer: Shivaram Venkataraman <shivaram@cs.berkeley.edu>
@@ -29,7 +29,8 @@ Collate:
     'client.R'
     'context.R'
     'deserialize.R'
+    'functions.R'
+    'mllib.R'
     'serialize.R'
     'sparkR.R'
     'utils.R'
-    'zzz.R'

R/pkg/NAMESPACE

@@ -10,6 +10,11 @@ export("sparkR.init")
 export("sparkR.stop")
 export("print.jobj")
 
+# MLlib integration
+exportMethods("glm",
+              "predict",
+              "summary")
+
 # Job group lifecycle management methods
 export("setJobGroup",
        "clearJobGroup",
@@ -22,7 +27,9 @@ exportMethods("arrange",
               "collect",
               "columns",
               "count",
+              "crosstab",
               "describe",
+              "dim",
               "distinct",
               "dropna",
               "dtypes",
@@ -39,11 +46,16 @@ exportMethods("arrange",
               "isLocal",
               "join",
               "limit",
-              "orderBy",
+              "merge",
               "mutate",
+              "na.omit",
               "names",
+              "ncol",
+              "nrow",
+              "orderBy",
               "persist",
               "printSchema",
+              "rbind",
               "registerTempTable",
               "rename",
               "repartition",
@@ -57,9 +69,13 @@ exportMethods("arrange",
               "selectExpr",
               "show",
               "showDF",
+              "subset",
               "summarize",
+              "summary",
               "take",
+              "transform",
               "unionAll",
+              "unique",
               "unpersist",
               "where",
               "withColumn",
@@ -68,58 +84,139 @@ exportMethods("arrange",
 
 exportClasses("Column")
 
-exportMethods("abs",
+exportMethods("%in%",
+              "abs",
               "acos",
+              "add_months",
               "alias",
               "approxCountDistinct",
               "asc",
+              "ascii",
               "asin",
               "atan",
               "atan2",
               "avg",
+              "base64",
+              "between",
+              "bin",
+              "bitwiseNOT",
               "cast",
               "cbrt",
+              "ceil",
               "ceiling",
+              "concat",
+              "concat_ws",
               "contains",
+              "conv",
               "cos",
               "cosh",
+              "count",
               "countDistinct",
+              "crc32",
+              "date_add",
+              "date_format",
+              "date_sub",
+              "datediff",
+              "dayofmonth",
+              "dayofyear",
               "desc",
               "endsWith",
               "exp",
+              "explode",
               "expm1",
+              "expr",
+              "factorial",
+              "first",
               "floor",
+              "format_number",
+              "format_string",
+              "from_unixtime",
+              "from_utc_timestamp",
               "getField",
               "getItem",
+              "greatest",
+              "hex",
+              "hour",
               "hypot",
+              "ifelse",
+              "initcap",
+              "instr",
+              "isNaN",
               "isNotNull",
               "isNull",
               "last",
+              "last_day",
+              "least",
+              "length",
+              "levenshtein",
               "like",
+              "lit",
+              "locate",
               "log",
               "log10",
               "log1p",
+              "log2",
               "lower",
+              "lpad",
+              "ltrim",
               "max",
+              "md5",
               "mean",
               "min",
+              "minute",
+              "month",
+              "months_between",
               "n",
               "n_distinct",
+              "nanvl",
+              "negate",
+              "next_day",
+              "otherwise",
+              "pmod",
+              "quarter",
+              "rand",
+              "randn",
+              "regexp_extract",
+              "regexp_replace",
+              "reverse",
               "rint",
               "rlike",
+              "round",
+              "rpad",
+              "rtrim",
+              "second",
+              "sha1",
+              "sha2",
+              "shiftLeft",
+              "shiftRight",
+              "shiftRightUnsigned",
               "sign",
+              "signum",
               "sin",
               "sinh",
+              "size",
+              "soundex",
               "sqrt",
               "startsWith",
               "substr",
+              "substring_index",
               "sum",
               "sumDistinct",
               "tan",
               "tanh",
               "toDegrees",
               "toRadians",
-              "upper")
+              "to_date",
+              "to_utc_timestamp",
+              "translate",
+              "trim",
+              "unbase64",
+              "unhex",
+              "unix_timestamp",
+              "upper",
+              "weekofyear",
+              "when",
+              "year")
 
 exportClasses("GroupedData")
 exportMethods("agg")