From b940bcddf07d2d3d02f212241a1c964f52210694 Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Fri, 26 Feb 2016 17:28:44 -0800 Subject: [PATCH 1/8] ZEPPELIN-700. Add Ansible R role to Virtual Machine to support dependencies for the R Interpreter --- .../vagrant/zeppelin-dev/ansible-roles.yml | 1 + .../zeppelin-dev/roles/r/defaults/main.yml | 9 +++ .../zeppelin-dev/roles/r/tasks/main.yml | 62 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml create mode 100644 scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml diff --git a/scripts/vagrant/zeppelin-dev/ansible-roles.yml b/scripts/vagrant/zeppelin-dev/ansible-roles.yml index ba2a8c8e25c..cc395227c86 100644 --- a/scripts/vagrant/zeppelin-dev/ansible-roles.yml +++ b/scripts/vagrant/zeppelin-dev/ansible-roles.yml @@ -23,3 +23,4 @@ - nodejs - maven - python-addons + - r diff --git a/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml new file mode 100644 index 00000000000..c064ad40cc2 --- /dev/null +++ b/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml @@ -0,0 +1,9 @@ +# defaults variables for r role +--- +r_cran_mirror: http://cran.rstudio.com/ + +r_repository: + - type: deb + url: "{{ r_cran_mirror }}/bin/linux/ubuntu {{ ansible_distribution_release }}/" + +r_packages_repos: "{{ r_cran_mirror }}" \ No newline at end of file diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml new file mode 100644 index 00000000000..297df96dcd1 --- /dev/null +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install R binaries and CRAN packages +--- + +# The Ubuntu archives on CRAN are signed with a key with ID E084DAB9. Add this key to the system. +- name: repository | add public key + apt_key: + id: E084DAB9 + keyserver: keyserver.ubuntu.com + state: present + +# in order to get the latest version of R, add CRAN repository to the to the list of sources +- name: repository | add cran-r + apt_repository: + repo: "{{ item.type }} {{ item.url }}" + state: present + update_cache: true + with_items: r_repository + +- name: Install R. This may take a while. + apt: pkg=r-base state=present + +- name: Install R packages required for the R interpreter. This may take a while. + shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" + with_items: + - evaluate + - knitr + +- name: Install R packages recommended for the R interpreter. + shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" + with_items: + - repr + - htmltools + - base64enc + +# requirements for Plotly include libcurl and libssl +- name: Install libcurl + apt: pkg={{ item }} state=present + with_items: + - libcurl4-openssl-dev + - libssl-dev + +- name: Install other helpful R packages + shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" + with_items: + - ggplot2 + - plotly + From ef8f638db4185be5eac8d3cc6c87b3c3eabef20b Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Fri, 26 Feb 2016 17:29:52 -0800 Subject: [PATCH 2/8] ZEPPELIN-700. Add Ansible R role to Virtual Machine to support dependencies for the R Interpreter --- .../zeppelin-dev/roles/r/defaults/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml index c064ad40cc2..0072470c6de 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/defaults/main.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # defaults variables for r role --- r_cran_mirror: http://cran.rstudio.com/ From fb18a2b0578ca843ffceb4c6b5c3956a2eabc6ff Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Mon, 29 Feb 2016 19:11:32 -0800 Subject: [PATCH 3/8] plotting in r interpreter requires the repr package. install devtools package first so repr can be installed. --- scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml index 297df96dcd1..20ba35a2582 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -39,11 +39,14 @@ with_items: - evaluate - knitr - + - devtools + +- name: Install R repr package recommended for the R interpreter display system (requires devtools first). + shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('repr' %in% installed.packages()[,'Package'])) devtools::install_github('IRkernel/repr')" + - name: Install R packages recommended for the R interpreter. shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" with_items: - - repr - htmltools - base64enc From 5643fc660eeed4f814ade57d3e1bc1895ac5a5b8 Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Mon, 29 Feb 2016 19:47:07 -0800 Subject: [PATCH 4/8] plotting in r interpreter requires the repr package. install devtools package first so repr can be installed. --- .../vagrant/zeppelin-dev/roles/r/tasks/main.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml index 20ba35a2582..2a3c2781ef0 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -49,17 +49,3 @@ with_items: - htmltools - base64enc - -# requirements for Plotly include libcurl and libssl -- name: Install libcurl - apt: pkg={{ item }} state=present - with_items: - - libcurl4-openssl-dev - - libssl-dev - -- name: Install other helpful R packages - shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" - with_items: - - ggplot2 - - plotly - From d4979944aa2825bdde6803e44da507324e567a80 Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Tue, 8 Mar 2016 08:41:02 -0800 Subject: [PATCH 5/8] fix so that devtools will install --- .../zeppelin-dev/roles/r/tasks/main.yml | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml index 2a3c2781ef0..96fe80d0ce2 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -26,7 +26,7 @@ # in order to get the latest version of R, add CRAN repository to the to the list of sources - name: repository | add cran-r apt_repository: - repo: "{{ item.type }} {{ item.url }}" + repo: "{{item.type}} {{item.url}}" state: present update_cache: true with_items: r_repository @@ -34,18 +34,30 @@ - name: Install R. This may take a while. apt: pkg=r-base state=present +- name: openssl and libcurl required for R devtools package + apt: pkg={{item}} state=present + with_items: + - libssl-dev + - libcurl4-openssl-dev + - name: Install R packages required for the R interpreter. This may take a while. shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" with_items: - - evaluate - - knitr - - devtools + - evaluate + - knitr + - devtools - name: Install R repr package recommended for the R interpreter display system (requires devtools first). shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('repr' %in% installed.packages()[,'Package'])) devtools::install_github('IRkernel/repr')" +- name: Install rCharts (requires devtools first). + shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('rCharts' %in% installed.packages()[,'Package'])) devtools::install_github('rCharts', 'ramnathv')" + - name: Install R packages recommended for the R interpreter. shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" with_items: - - htmltools - - base64enc + - htmltools + - base64enc + - ggplot2 + - googleVis + From bdcdf5f97d094bcd369ca2ebee5f0b418f7a3a89 Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Tue, 8 Mar 2016 11:32:25 -0800 Subject: [PATCH 6/8] removed packages not required for pr702. repr not needed - base64encode not needed - htmltools not needed --- scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml index 96fe80d0ce2..13d5a557005 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -40,24 +40,19 @@ - libssl-dev - libcurl4-openssl-dev +# knitr will also pull in the `evaluate` R package as a dependency - name: Install R packages required for the R interpreter. This may take a while. shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" with_items: - - evaluate - knitr - devtools -- name: Install R repr package recommended for the R interpreter display system (requires devtools first). - shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('repr' %in% installed.packages()[,'Package'])) devtools::install_github('IRkernel/repr')" - - name: Install rCharts (requires devtools first). shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('rCharts' %in% installed.packages()[,'Package'])) devtools::install_github('rCharts', 'ramnathv')" - name: Install R packages recommended for the R interpreter. shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" with_items: - - htmltools - - base64enc - ggplot2 - googleVis From a9a2052bf48063cb28cd8906f874162bedfe5763 Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Thu, 10 Mar 2016 14:47:40 -0800 Subject: [PATCH 7/8] add base64enc, repr and htmltools --- scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml index 13d5a557005..e00dba3891f 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -50,9 +50,15 @@ - name: Install rCharts (requires devtools first). shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('rCharts' %in% installed.packages()[,'Package'])) devtools::install_github('rCharts', 'ramnathv')" +- name: Install R repr package recommended for the R interpreter display system (requires devtools first). + shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('repr' %in% installed.packages()[,'Package'])) devtools::install_github('IRkernel/repr')" + - name: Install R packages recommended for the R interpreter. shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))" with_items: - ggplot2 - googleVis + - mplot + - htmltools + - base64enc From e03dba58891bd7b8b633e00a6be4e971495c711a Mon Sep 17 00:00:00 2001 From: Jeff Steinmetz Date: Mon, 28 Mar 2016 14:35:16 -0700 Subject: [PATCH 8/8] update to support R interpreter sparkr build profile --- scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml | 1 + scripts/vagrant/zeppelin-dev/show-instructions.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml index e00dba3891f..54285d8ae8c 100644 --- a/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml +++ b/scripts/vagrant/zeppelin-dev/roles/r/tasks/main.yml @@ -61,4 +61,5 @@ - mplot - htmltools - base64enc + - data.table diff --git a/scripts/vagrant/zeppelin-dev/show-instructions.sh b/scripts/vagrant/zeppelin-dev/show-instructions.sh index 43bba768245..16a399d31f7 100644 --- a/scripts/vagrant/zeppelin-dev/show-instructions.sh +++ b/scripts/vagrant/zeppelin-dev/show-instructions.sh @@ -32,9 +32,9 @@ echo echo 'cd /vagrant/incubator-zeppelin' echo 'mvn clean package -DskipTests' echo -echo '# or for a specific build' +echo '# or for a specific Spark/Hadoop build with additional options such as python and R support' echo -echo 'mvn clean package -Pspark-1.5 -Ppyspark -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests' +echo 'mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests' echo './bin/zeppelin-daemon.sh start' echo echo 'On your host machine browse to http://localhost:8080/'