cluster.conf

name: sparklyr-conf
provider {
    type: aws
    accessKeyId: ${?AWS_ACCESS_KEY_ID}
    secretAccessKey: ${?AWS_SECRET_ACCESS_KEY}
    region: ap-northeast-1 # Change to your preferable reagion
    subnetId: <SUBNET-REPLACE-ME>
    securityGroupsIds: <SECURITY-GROUP-REPLACE-ME>
    instanceNamePrefix: cloudera-director-sparklyr
    # associatePublicIpAddresses: true
}
ssh {
    username: centos # for CentOS Image https://aws.amazon.com/marketplace/pp/B00O7WM7QW
    privateKey: <PATH-TO-PEM-FILE-REPLACE-ME> # with an absolute path to .pem file
}
common-instance-properties {
    image: ami-eec1c380
    tags {
        owner: ${?USER}
    }
}

instances {
    m42x : ${common-instance-properties} {
        type: m4.2xlarge   # requires an HVM AMI
    }
    d2x : ${common-instance-properties} {
        type: d2.xlarge
    }

    t2l : ${common-instance-properties} {
      type: t2.large
      bootstrapScript: """#!/bin/sh
# Flag for gateway node
echo "GATEWAY" >> /root/gateway

exit 0

"""
    }
}
cloudera-manager {

    instance: ${instances.m42x} {
        tags {
            application: "Cloudera Manager 5"
        }
    }
    bootstrapScript: """#!/bin/sh
yum -y install centos-release-scl wget
cd /tmp

wget 'http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.0.0.cloudera2.jar'

mkdir -p /opt/cloudera/csd
chown cloudera-scm:cloudera-scm SPARK*.jar
chmod 644 SPARK*.jar
mv SPARK*.jar /opt/cloudera/csd/

exit 0
"""

    enableEnterpriseTrial: true

    repository: "http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.10/"
    repositoryKeyUrl: "http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera"

}
cluster {
    products {
      CDH: 5 # includes Spark
      SPARK2: 2
    }

    # S3 Configurations
    configs {
      HDFS {
        core_site_safety_valve: """
            <property>
                <name>fs.s3a.access.key</name>
                <value><REPLACE-ME-AWS-ACCESS-KEY-ID></value>
            </property>
            <property>
                <name>fs.s3a.secret.key</name>
                <value><REPLACE-ME-AWS-SECRET-ACCESS-KEY></value>
            </property>
            <property>
                <name>fs.s3a.block.size</name>
                <value>134217728</value>
            </property>
            <property>
              <name>fs.s3a.server-side-encryption-algorithm</name>
              <value>AES256</value>
            </property>
            <property>
              <name>fs.s3a.connection.ssl.enabled</name>
              <value>true</value>
              <description>Enables or disables SSL connections to S3.</description>
            </property>
        """
      }
    }

    parcelRepositories: [
      "http://archive.cloudera.com/cdh5/parcels/5.10/",
      "http://archive.cloudera.com/spark2/parcels/2.0/",
    ]

    services: [HDFS, YARN, ZOOKEEPER, HUE, HIVE, OOZIE, SPARK_ON_YARN]
    masters {
        count: 1
        instance: ${instances.m42x} {
            tag {
                group: master
            }
        }

        roles {
            HDFS: [NAMENODE, SECONDARYNAMENODE]
            YARN: [RESOURCEMANAGER, JOBHISTORY]
            ZOOKEEPER: [SERVER]
            HIVE: [HIVESERVER2, HIVEMETASTORE]
            HUE: [HUE_SERVER]
            OOZIE: [OOZIE_SERVER]
            SPARK2_ON_YARN: [SPARK2_YARN_HISTORY_SERVER]
        }
    }

    workers {
      count: 3
      minCount: 3

      instance: ${instances.m42x} {
        tags {
          group: worker
        }
      }

      roles {
        HDFS: [DATANODE]
        YARN: [NODEMANAGER]
      }
    }
    gateways {
      count: 1

      instance: ${instances.t2l} {
        tags {
          group: gateway
        }
      }

      roles {
        HDFS: [GATEWAY]
        HIVE: [GATEWAY]
        SPARK2_ON_YARN: [GATEWAY]
      }
    }
    instancePostCreateScripts: ["""#!/bin/sh
echo 'Hello World!'
exit 0
    """,
    """#!/bin/sh
set -ex

sudo useradd rsuser
sudo sh -c "echo 'rsuser:cloudera' | chpasswd"

if [ ! -e /root/gateway ]; then
exit 0
fi

#install R
sudo yum install -y epel-release
sudo yum install -y wget texlive texlive-epsf texinfo texinfo-tex libcurl-devel R

#install rstudio-server
wget https://download2.rstudio.org/rstudio-server-rhel-1.0.44-x86_64.rpm
sudo yum -y install --nogpgcheck rstudio-server-rhel-1.0.44-x86_64.rpm
sudo service rstudio-server start

#setup SPARK_HOME env variable
sudo sh -c "echo 'SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark/' >> /usr/lib64/R/etc/Renviron"

echo "Installing R packages"
#install packages
# Rscript -e 'update.packages(ask = FALSE)'
for pkgname in sparklyr nycflights13 ggplot2 dplyr maps geosphere
do
    sudo Rscript -e 'if(!require("'$pkgname'", character.only = TRUE, quietly = TRUE)) install.packages("'$pkgname'", dependencies = TRUE, repos="https://cran.r-project.org")'
done

sudo -u hdfs hdfs dfs -mkdir /user/rsuser
sudo -u hdfs hdfs dfs -chown rsuser:rsuser /user/rsuser

# If you have already created table on S3, you can create external table with following code:
#hcat -e '''
#CREATE EXTERNAL TABLE airlines_bi_pq (
#  year INT,
#  month INT,
#  day INT,
#  dayofweek INT,
#  dep_time INT,
#  crs_dep_time INT,
#  arr_time INT,
#  crs_arr_time INT,
#  carrier STRING,
#  flight_num INT,
#  tail_num INT,
#  actual_elapsed_time INT,
#  crs_elapsed_time INT,
#  airtime INT,
#  arrdelay INT,
#  depdelay INT,
#  origin STRING,
#  dest STRING,
#  distance INT,
#  taxi_in INT,
#  taxi_out INT,
#  cancelled INT,
#  cancellation_code STRING,
#  diverted INT,
#  carrier_delay INT,
#  weather_delay INT,
#  nas_delay INT,
#  security_delay INT,
#  late_aircraft_delay INT,
#  date_yyyymm STRING
#) STORED AS PARQUET
#LOCATION "s3a://<YOUR-BUCKET-NAME>/data/refined/www.ibis-project.org/airlines_bi_pq"
#TBLPROPERTIES ("numFiles"="0", "COLUMN_STATS_ACCURATE"="true", "transient_lastDdlTime"="1484553764", "numRows"="123534969", "totalSize"="0", "rawDataSize"="-1");
#
#CREATE EXTERNAL TABLE airports (
#  iata STRING,
#  name STRING,
#  city STRING,
#  state STRING,
#  country STRING,
#  latitude DOUBLE,
#  longitude DOUBLE
#)
#ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.OpenCSVSerde"
#STORED AS TEXTFILE
#LOCATION "s3a://<YOUR-BUCKET-NAME>/data/raw/openflights.org/airports"
#tblproperties("skip.header.line.count"="1");
#
## If you have Parquet format file, you can use following query.
#CREATE EXTERNAL TABLE airports_pq (
#  iata STRING,
#  name STRING,
#  city STRING,
#  state STRING,
#  country STRING,
#  latitude DOUBLE,
#  longitude DOUBLE
#) STORED AS PARQUET LOCATION "s3a://<YOUR-BUCKET-NAME>/data/refined/openflights.org/airports_pq"
#TBLPROPERTIES ("transient_lastDdlTime"="1485426798");
#'''


exit 0
    """]
}