From c4e996e11cedd6e186ad09ed27ca2035a5d71a0e Mon Sep 17 00:00:00 2001 From: Evans Ye Date: Sun, 15 Jan 2017 03:41:29 +0800 Subject: [PATCH] BIGTOP-2668. Polish and update Docker Provisioner configurations --- provisioner/docker/README.md | 13 +++----- provisioner/docker/config.yaml | 33 +------------------ provisioner/docker/config_centos6.yaml | 25 ++++++++++++++ provisioner/docker/config_centos7.yaml | 13 ++------ ...config_debian.yaml => config_debian8.yaml} | 13 ++------ ..._ubuntu.yaml => config_ubuntu_trusty.yaml} | 13 ++------ provisioner/docker/docker-compose.yml | 1 + provisioner/docker/docker-hadoop.sh | 3 +- 8 files changed, 43 insertions(+), 71 deletions(-) mode change 100644 => 120000 provisioner/docker/config.yaml create mode 100644 provisioner/docker/config_centos6.yaml rename provisioner/docker/{config_debian.yaml => config_debian8.yaml} (77%) rename provisioner/docker/{config_ubuntu.yaml => config_ubuntu_trusty.yaml} (77%) diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md index 8a8b8ee5b9..ef52ff1afd 100644 --- a/provisioner/docker/README.md +++ b/provisioner/docker/README.md @@ -123,20 +123,17 @@ usage: docker-hadoop.sh [-C file ] args ``` docker: - memory_size: "2048" + memory_limit: "2g" ``` -2) Use different host ports mapping for web UIs +2) Enable local repository -``` -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" +If you've built packages using local cloned bigtop and produced the apt/yum repo, set the following to true to deploy those packages: ``` -Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed - +enable_local_repo = true +``` ##Configure Apache Hadoop ecosystem components * Choose the ecosystem you want to be deployed by modifying components in config.yaml diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml deleted file mode 100644 index 86f35e75cd..0000000000 --- a/provisioner/docker/config.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:centos-6" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64" -distro: centos -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml new file mode 120000 index 0000000000..776a8f1f0e --- /dev/null +++ b/provisioner/docker/config.yaml @@ -0,0 +1 @@ +config_centos7.yaml \ No newline at end of file diff --git a/provisioner/docker/config_centos6.yaml b/provisioner/docker/config_centos6.yaml new file mode 100644 index 0000000000..54ca86c304 --- /dev/null +++ b/provisioner/docker/config_centos6.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:centos-6" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64" +distro: centos +components: [hadoop, yarn] +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml index 3610a8f17d..44920f4b20 100644 --- a/provisioner/docker/config_centos7.yaml +++ b/provisioner/docker/config_centos7.yaml @@ -14,19 +14,12 @@ # limitations under the License. docker: - memory_size: "4096" - image: "bigtop/deploy:centos-7" + memory_limit: "4g" + image: "bigtop/puppet:centos-7" -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64" +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64" distro: centos components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" enable_local_repo: false smoke_test_components: [mapreduce, pig] jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/config_debian.yaml b/provisioner/docker/config_debian8.yaml similarity index 77% rename from provisioner/docker/config_debian.yaml rename to provisioner/docker/config_debian8.yaml index f6b11aea7d..2ed894635b 100644 --- a/provisioner/docker/config_debian.yaml +++ b/provisioner/docker/config_debian8.yaml @@ -14,19 +14,12 @@ # limitations under the License. docker: - memory_size: "4096" - image: "bigtop/deploy:debian-8" + memory_limit: "4g" + image: "bigtop/puppet:debian-8" -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64" +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64" distro: debian components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" enable_local_repo: false smoke_test_components: [mapreduce, pig] jdk: "openjdk-7-jdk" diff --git a/provisioner/docker/config_ubuntu.yaml b/provisioner/docker/config_ubuntu_trusty.yaml similarity index 77% rename from provisioner/docker/config_ubuntu.yaml rename to provisioner/docker/config_ubuntu_trusty.yaml index e4ea6f3b1d..4ac83dc370 100644 --- a/provisioner/docker/config_ubuntu.yaml +++ b/provisioner/docker/config_ubuntu_trusty.yaml @@ -14,19 +14,12 @@ # limitations under the License. docker: - memory_size: "4096" - image: "bigtop/deploy:ubuntu-14.04" + memory_limit: "4g" + image: "bigtop/puppet:ubuntu-14.04" -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64" +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64" distro: debian components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" enable_local_repo: false smoke_test_components: [mapreduce, pig] jdk: "openjdk-7-jdk" diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml index df107aea26..cee849e934 100644 --- a/provisioner/docker/docker-compose.yml +++ b/provisioner/docker/docker-compose.yml @@ -18,6 +18,7 @@ bigtop: command: /sbin/init domainname: bigtop.apache.org privileged: true + mem_limit: ${MEM_LIMIT} volumes: - ../../:/bigtop-home - ./config/hiera.yaml:/etc/puppet/hiera.yaml diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index d40bd8b776..05b712c54c 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -43,7 +43,6 @@ create() { mkdir -p config/hieradata 2> /dev/null echo > ./config/hiera.yaml echo > ./config/hosts - export DOCKER_IMAGE=$(get-yaml-config docker image) # Startup instances docker-compose -p $PROVISION_ID scale bigtop=$1 @@ -193,6 +192,8 @@ fi if [ -n "$PROVISION_ID" ]; then NODES=(`docker-compose -p $PROVISION_ID ps -q`) fi +export DOCKER_IMAGE=$(get-yaml-config docker image) +export MEM_LIMIT=$(get-yaml-config docker memory_limit) while [ $# -gt 0 ]; do case "$1" in