BIGTOP-2646. Move Vagrant Provisioner from bigtop-deploy/vm to provisioner directory

provisioner/utils/setup-env-centos.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+yum -y install rng-tools
+if [ -x /usr/bin/systemctl ] ; then
+    sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service
+    systemctl daemon-reload
+    systemctl start rngd
+else
+    sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
+    service rngd start
+fi
+
+if [ $enable_local_repo == "true" ]; then
+    echo "Enabling local yum."
+    yum -y install yum-utils
+    sudo echo "gpgcheck=0" >> /etc/yum.conf
+    sudo yum-config-manager --add-repo file:///bigtop-home/output
+else
+    echo "local yum = $enable_local_repo ; NOT Enabling local yum.  Packages will be pulled from remote..."
+fi
+

provisioner/utils/setup-env-debian.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+apt-get -y install rng-tools
+sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
+service rng-tools start
+
+if [ $enable_local_repo == "true" ]; then
+    echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
+    apt-get update
+else
+    apt-get install -y apt-transport-https
+    echo "local apt = $enable_local_repo ; NOT Enabling local apt. Packages will be pulled from remote..."
+fi
+

provisioner/utils/smoke-tests.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HCFS_USER="hdfs"
+SMOKE_TESTS=${1:-mapreduce,pig}
+
+# Autodetect JAVA_HOME
+if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+else
+  >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
+  exit 2
+fi
+
+echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
+
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
+export HIVE_HOME=/usr/lib/hive/
+export PIG_HOME=/usr/lib/pig/
+export FLUME_HOME=/usr/lib/flume/
+export SQOOP_HOME=/usr/lib/sqoop/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export MAHOUT_HOME="/usr/lib/mahout"
+
+prep() {
+    HADOOP_COMMAND=$1
+    su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir /user/vagrant /user/root"
+    su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant"
+    su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root"
+}
+
+prep hadoop
+if [[ $SMOKE_TESTS == *"qfs"* ]]; then
+    prep hadoop-qfs
+fi
+
+if [ -f /etc/debian_version ] ; then
+    apt-get -y install pig hive flume mahout sqoop
+else
+    yum install -y pig hive flume mahout sqoop
+fi
+ALL_SMOKE_TASKS=""
+for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
+  ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
+done
+cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
+# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
+# permissions as used for smoke-tests execution
+rm -rf buildSrc/build/test-results/binary
+

provisioner/vagrant/.gitignore

@@ -0,0 +1 @@
+config

provisioner/vagrant/README.md

@@ -0,0 +1,99 @@
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements. See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+#BigTop VM provisioner
+
+## Overview
+
+The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
+
+- to test bigtop smoke tests
+- to test bigtop puppet recipes
+
+Eventually, we may also add ability to build AND provision bigtop in a vagrant recipe, which would essentially
+give full validation of the BigTop stack.
+
+## USAGE
+
+1) Install [Vagrant](https://www.vagrantup.com/downloads.html)(DO NOT install 1.8.5 because of a critical [bug](https://github.com/mitchellh/vagrant/issues/7631))
+
+2) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to manage `/etc/hosts`
+
+```
+vagrant plugin install vagrant-hostmanager
+```
+
+3) (Optional) Install [vagrant-cachier plugin](https://github.com/fgrehm/vagrant-cachier) to cache packages at local
+
+```
+vagrant plugin install vagrant-cachier
+```
+
+4) To provision a one node Apache Hadoop cluster on top of vagrant boxes
+
+```
+vagrant up
+```
+
+5) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml
+
+```
+num_instances: 5
+```
+
+6) Test on local built packages is available by:
+
+first, build up local yum repo
+
+```
+cd bigtop; ./gradlew alluxio-yum
+```
+
+and then enable local yum in vagrantconfig.yaml
+
+```
+enable_local_repo: true
+```
+
+finally, spin up the cluster
+```
+vagrant up
+```
+
+## Override Vagrant configurations
+You can override vagrant configurations from environment variables:
+```
+$ export REPO=http://repo.example.com/path/to
+$ vagrant up
+```
+
+or
+
+```
+$ MEMORY_SIZE=8000 vagrant up
+```
+
+##Configure Apache Hadoop ecosystem components
+* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml
+
+```
+components: "hadoop, hbase, yarn,..."
+```
+
+By default, Apache Hadoop and YARN will be installed.
+See `bigtop-deploy/puppet/hieradata/site.yaml` for more details.