Merge remote-tracking branch 'upstream/master' into SPARK-29721-2

apache · Feb 29, 2020 · 3928eb0 · 3928eb0
2 parents 0383592 + 0e2ca11
commit 3928eb0
Show file tree

Hide file tree

Showing 661 changed files with 14,895 additions and 8,441 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
@@ -6,6 +6,8 @@ Thanks for sending a pull request!  Here are some tips for you:
   4. Be sure to keep the PR description updated to reflect all changes.
   5. Please write your PR title to summarize what this PR proposes.
   6. If possible, provide a concise example to reproduce the issue for a faster review.
+  7. If you want to add a new configuration, please read the guideline first for naming configurations in
+     'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
 -->
 
 ### What changes were proposed in this pull request?

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -117,3 +117,40 @@ jobs:
       run: ./R/install-dev.sh
     - name: lint-r
       run: ./dev/lint-r
+
+  docs:
+    runs-on: ubuntu-latest
+    name: Generate documents
+    steps:
+    - uses: actions/checkout@master
+    - uses: actions/cache@v1
+      with:
+        path: ~/.m2/repository
+        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          docs-maven-repo-
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '1.8'
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+        architecture: 'x64'
+    - uses: actions/setup-ruby@v1
+      with:
+        ruby-version: '2.7'
+    - name: Install R
+      run: |
+        echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' | sudo tee -a /etc/apt/sources.list
+        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
+        sudo apt-get update
+        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
+    - name: Install packages
+      run: |
+        pip install sphinx mkdocs numpy
+        gem install jekyll jekyll-redirect-from rouge
+        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
+    - name: Run jekyll build
+      run: |
+        cd docs
+        jekyll build
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.0
+Version: 3.1.0
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),

diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
@@ -199,9 +199,13 @@ exportMethods("%<=>%",
               "approx_count_distinct",
               "approxCountDistinct",
               "approxQuantile",
+              "array_aggregate",
               "array_contains",
               "array_distinct",
               "array_except",
+              "array_exists",
+              "array_filter",
+              "array_forall",
               "array_intersect",
               "array_join",
               "array_max",
@@ -210,9 +214,11 @@ exportMethods("%<=>%",
               "array_remove",
               "array_repeat",
               "array_sort",
+              "array_transform",
               "arrays_overlap",
               "array_union",
               "arrays_zip",
+              "arrays_zip_with",
               "asc",
               "ascii",
               "asin",
@@ -314,10 +320,12 @@ exportMethods("%<=>%",
               "ltrim",
               "map_concat",
               "map_entries",
+              "map_filter",
               "map_from_arrays",
               "map_from_entries",
               "map_keys",
               "map_values",
+              "map_zip_with",
               "max",
               "md5",
               "mean",
@@ -396,6 +404,8 @@ exportMethods("%<=>%",
               "to_timestamp",
               "to_utc_timestamp",
               "translate",
+              "transform_keys",
+              "transform_values",
               "trim",
               "trunc",
               "unbase64",