From 23f56d0351d1ed8d92d0923ecfd172bcac60dca5 Mon Sep 17 00:00:00 2001 From: Evans Ye Date: Sat, 23 Jan 2016 10:33:03 +0000 Subject: [PATCH] BIGTOP-2254. Replace the Docker orchestration tool from Vagrant to Docker Compose --- provisioner/docker/.gitignore | 1 + provisioner/docker/README.md | 160 +++++++++++++++++ provisioner/docker/docker-compose.yml | 10 ++ provisioner/docker/docker-hadoop.sh | 179 +++++++++++++++++++ provisioner/docker/provision.sh | 36 ++++ provisioner/docker/vagrantconfig.yaml | 32 ++++ provisioner/docker/vagrantconfig_debian.yaml | 32 ++++ provisioner/docker/vagrantconfig_ubuntu.yaml | 32 ++++ provisioner/utils/setup-env-centos.sh | 36 ++++ provisioner/utils/setup-env-debian.sh | 34 ++++ provisioner/utils/smoke-tests.sh | 56 ++++++ 11 files changed, 608 insertions(+) create mode 100644 provisioner/docker/.gitignore create mode 100644 provisioner/docker/README.md create mode 100644 provisioner/docker/docker-compose.yml create mode 100755 provisioner/docker/docker-hadoop.sh create mode 100755 provisioner/docker/provision.sh create mode 100644 provisioner/docker/vagrantconfig.yaml create mode 100644 provisioner/docker/vagrantconfig_debian.yaml create mode 100644 provisioner/docker/vagrantconfig_ubuntu.yaml create mode 100755 provisioner/utils/setup-env-centos.sh create mode 100755 provisioner/utils/setup-env-debian.sh create mode 100755 provisioner/utils/smoke-tests.sh diff --git a/provisioner/docker/.gitignore b/provisioner/docker/.gitignore new file mode 100644 index 0000000000..f733c4b5fb --- /dev/null +++ b/provisioner/docker/.gitignore @@ -0,0 +1 @@ +config/ diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md new file mode 100644 index 0000000000..79e50f7851 --- /dev/null +++ b/provisioner/docker/README.md @@ -0,0 +1,160 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------------------------------------------------------------ + +#BigTop Docker provisioner + +## Overview + +The Vagrantfile definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories. +This cluster can be used: + +- to test bigtop smoke tests +- to test bigtop puppet recipes + +These containers start sshd daemons, which vagrant uses to provision and install the hadoop cluster. + +This has been verified on docker client 1.2.0, with api version 1.15, and vagrant 1.6.5 on Fedora 20 as well as Centos 6. + +## Prerequisites + +### OS X and Windows + +* Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) + +* Install [Vagrant](http://www.vagrantup.com/downloads.html). Need version 1.6.5 or higher. + +### Linux + +* [Kernel Requirements](http://docker.readthedocs.org/en/v0.5.3/installation/kernel/) + +* Install [Docker](https://docs.docker.com/installation/) + +* Install [Vagrant](http://www.vagrantup.com/downloads.html) + +## Getting Started + +* Create a 3 node Bigtop Hadoop cluster from scratch + +NOTE : SELinux can PREVENT you from ssh'ing into your docker container. +As a brute force way to disable it - remove it from vi /etc/sysconfig/docker arguments +(fedora and centos may by default launch docker daemon with the --selinux-enabled option)! +In the future, lets update this README with the RIGHT way to allow selinux without breaking +ssh into a docker container! + +``` +service docker restart +docker pull bigtop/deploy:centos-6 +``` +Now, you can start your cluster: + +In case of errors you can attempt running as root, or else, ping the mailing list. + +## USAGE + +1) Create a Bigtop Hadoop cluster by given # of node. (will place a file called config.rb) + +``` +./docker-hadoop.sh --create 3 +``` + +2) Destroy the cluster. + +``` +./docker-hadoop.sh --destroy +``` + +3) Update your cluster after doing configuration changes. (re-run puppet apply) + +``` +./docker-hadoop.sh --provision +``` + +4) Run Bigtop smoke tests + +``` +./docker-hadoop.sh --smoke-tests +``` + +5) Chain your operations with-in one command. + +``` +./docker-hadoop.sh --create 5 --smoke-tests --destroy +``` + +Commands will be executed by following order: + +``` +create 5 node cluster => run smoke tests => destroy the cluster +``` + +6) See helper message: + +``` +./docker-hadoop.sh -h +usage: docker-hadoop.sh [options] + -c NUM_INSTANCES, --create=NUM_INSTANCES Create a docker based Bigtop Hadoop cluster + -p, --provision Deploy configuration changes + -s, --smoke-tests Run Bigtop smoke tests + -d, --destroy Destroy the cluster + -h, --help + +``` + +##Configurations + +* There are several parameters can be configured in the vagrantconfig.yaml: + +1) Modify memory limit for Docker containers + +``` +docker: + memory_size: "2048" + +``` + +2) If you're running Docker provisioner on OS X or Windows, you can customize the boot2docker VM settings + +``` +boot2docker: + memory_size: "4096" + number_cpus: "2" +``` + +3) Use different host ports mapping for web UIs + +``` +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" + +``` +Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed + + +##Configure Apache Hadoop ecosystem components +* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml + +``` +components: "hadoop,hbase,yarn,..." +``` + +By default, Apache Hadoop, YARN, and Apache HBase will be installed. +See `bigtop-deploy/puppet/config/site.csv.example` for more details. + +##Notes + +* Users currently using vagrant 1.6+ is strongly recommanded to upgrade to 1.6.4+, otherwise you will encounter the [issue](https://github.com/mitchellh/vagrant/issues/3769) when installing plguins diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml new file mode 100644 index 0000000000..95831d1005 --- /dev/null +++ b/provisioner/docker/docker-compose.yml @@ -0,0 +1,10 @@ +bigtop: + image: ${DOCKER_IMAGE} + command: sleep infinity + domainname: bigtop.apache.org + privileged: true + volumes: + - ../../:/bigtop-home + - ./config/hiera.yaml:/etc/puppet/hiera.yaml + - ./config/hieradata:/etc/puppet/hieradata + - ./config/hosts:/etc/hosts diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh new file mode 100755 index 0000000000..5f07723ddf --- /dev/null +++ b/provisioner/docker/docker-hadoop.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet + +usage() { + echo "usage: $PROG [-C file ] args" + echo " -C file Use alternate file for vagrantconfig.yaml" + echo " commands:" + echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster" + echo " -p, --provision Deploy configuration changes" + echo " -s, --smoke-tests Run Bigtop smoke tests" + echo " -d, --destroy Destroy the cluster" + echo " -h, --help" + exit 1 +} + +create() { + # Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon + mkdir config 2> /dev/null + cat /dev/null > ./config/hiera.yaml + cat /dev/null > ./config/hosts + export DOCKER_IMAGE=$(get-yaml-config docker image) + + # Startup instances + docker-compose scale bigtop=$1 + if [ $? -ne 0 ]; then + echo "Docker container(s) startup failed!"; + exit 1; + fi + + # Get the headnode FQDN + nodes=(`docker-compose ps -q`) + hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}` + + # Fetch configurations form specificed yaml config file + repo=$(get-yaml-config repo) + components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]" + jdk=$(get-yaml-config jdk) + distro=$(get-yaml-config distro) + enable_local_repo=$(get-yaml-config enable_local_repo) + generate-config "$hadoop_head_node" "$repo" "$components" "$jdk" + + # Start provisioning + generate-hosts + bootstrap $distro $enable_local_repo + provision +} + +generate-hosts() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts + done + wait + +} + +generate-config() { + echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata" + cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml + yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/ + cat > ./config/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: $1 +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: $2 +hadoop_cluster_node::cluster_components: $3 +bigtop::jdk_package_name: $4 +EOF +} + +copy-to-instances() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + docker cp $1 $node:$2 & + done + wait +} + +bootstrap() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" & + done + wait +} + +provision() { + nodes=(`docker-compose ps -q`) + for node in ${nodes[*]}; do + bigtop-puppet $node & + done + wait +} + +smoke-tests() { + nodes=(`docker-compose ps -q`) + hadoop_head_node=${nodes:0:12} + smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`" + docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components" +} + +destroy() { + docker-compose stop + docker-compose rm -f + rm -rvf ./config +} + +bigtop-puppet() { + docker exec $1 bash -c 'puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp' +} + +get-yaml-config() { + RUBY_EXE=ruby + if [ $# -eq 1 ]; then + RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];" + elif [ $# -eq 2 ]; then + RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1']['$2'];" + else + echo "The yaml config retrieval function can only take 1 or 2 parameters."; + exit 1; + fi + cat ${vagrantyamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r' +} + +PROG=`basename $0` + +if [ $# -eq 0 ]; then + usage +fi + +vagrantyamlconf="vagrantconfig.yaml" +while [ $# -gt 0 ]; do + case "$1" in + -c|--create) + if [ $# -lt 2 ]; then + echo "Create requires a number" 1>&2 + usage + fi + create $2 + shift 2;; + -C|--conf) + if [ $# -lt 2 ]; then + echo "Alternative config file for vagrantconfig.yaml" 1>&2 + usage + fi + vagrantyamlconf=$2 + shift 2;; + -p|--provision) + provision + shift;; + -s|--smoke-tests) + smoke-tests + shift;; + -d|--destroy) + destroy + shift;; + -h|--help) + usage + shift;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done diff --git a/provisioner/docker/provision.sh b/provisioner/docker/provision.sh new file mode 100755 index 0000000000..6e40c6905d --- /dev/null +++ b/provisioner/docker/provision.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sysctl kernel.hostname=`hostname -f` + +# Unmount device /etc/hosts and replace it by a shared hosts file +echo -e "`hostname -i`\t`hostname -f`" >> /vagrant/hosts +umount /etc/hosts +mv /etc/hosts /etc/hosts.bak +ln -s /vagrant/hosts /etc/hosts + +# Prepare puppet configuration file +mkdir -p /etc/puppet/hieradata +cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet +cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ +cat > /etc/puppet/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: $1 +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: $2 +hadoop_cluster_node::cluster_components: $3 +bigtop::jdk_package_name: $4 +EOF diff --git a/provisioner/docker/vagrantconfig.yaml b/provisioner/docker/vagrantconfig.yaml new file mode 100644 index 0000000000..86f35e75cd --- /dev/null +++ b/provisioner/docker/vagrantconfig.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:centos-6" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64" +distro: centos +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/vagrantconfig_debian.yaml b/provisioner/docker/vagrantconfig_debian.yaml new file mode 100644 index 0000000000..f6b11aea7d --- /dev/null +++ b/provisioner/docker/vagrantconfig_debian.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:debian-8" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64" +distro: debian +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "openjdk-7-jdk" diff --git a/provisioner/docker/vagrantconfig_ubuntu.yaml b/provisioner/docker/vagrantconfig_ubuntu.yaml new file mode 100644 index 0000000000..e4ea6f3b1d --- /dev/null +++ b/provisioner/docker/vagrantconfig_ubuntu.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_size: "4096" + image: "bigtop/deploy:ubuntu-14.04" + +boot2docker: + memory_size: "4096" + number_cpus: "1" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64" +distro: debian +components: [hadoop, yarn] +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "openjdk-7-jdk" diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh new file mode 100755 index 0000000000..bb25d286a5 --- /dev/null +++ b/provisioner/utils/setup-env-centos.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +yum -y install rng-tools +sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd +service rngd start + +if [ $enable_local_repo == "true" ]; then + echo "Enabling local yum." + yum -y install yum-utils + sudo echo "gpgcheck=0" >> /etc/yum.conf + sudo yum-config-manager --add-repo file:///bigtop-home/output +else + echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." +fi + diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh new file mode 100755 index 0000000000..c5768d9af4 --- /dev/null +++ b/provisioner/utils/setup-env-debian.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +apt-get -y install rng-tools +sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools +service rng-tools start + +if [ $enable_local_repo == "true" ]; then + echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list + apt-get update +else + echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." +fi + diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh new file mode 100755 index 0000000000..8dac31c06e --- /dev/null +++ b/provisioner/utils/smoke-tests.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HCFS_USER="hdfs" +SMOKE_TESTS=${1:-mapreduce,pig} + +# Autodetect JAVA_HOME +if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +else + >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n" + exit 2 +fi + +echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export SQOOP_HOME=/usr/lib/sqoop/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export MAHOUT_HOME="/usr/lib/mahout" + +su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' + +if [ -f /etc/debian_version ] ; then + apt-get -y install pig hive flume mahout sqoop +else + yum install -y pig hive flume mahout sqoop +fi +ALL_SMOKE_TASKS="" +for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do + ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test" +done +cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info +# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same +# permissions as used for smoke-tests execution +rm -rf buildSrc/build/test-results/binary +