From ade7b721af7f01032e6839d23e0c82698fe9e68c Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Tue, 24 May 2016 14:51:48 +0200 Subject: [PATCH] BIGTOP-2357: Add Apache Flink to Bigtop / Create puppet recipes --- .../puppet/hieradata/bigtop/cluster.yaml | 9 ++- bigtop-deploy/puppet/hieradata/site.yaml | 1 + bigtop-deploy/puppet/manifests/cluster.pp | 5 ++ .../puppet/modules/flink/manifests/init.pp | 62 +++++++++++++++++++ .../modules/flink/templates/flink-conf.yaml | 31 ++++++++++ .../vm/vagrant-puppet-vm/Vagrantfile | 1 + 6 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 bigtop-deploy/puppet/modules/flink/manifests/init.pp create mode 100644 bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index f592f64c4f..5c2c5f1274 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -16,7 +16,7 @@ # "$components" list. If $components isn't set then everything in the stack will # be installed as usual. Otherwise only a specified list will be set # Possible elements: -# hadoop,yarn,hbase,alluxio,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs, +# hadoop,yarn,hbase,alluxio,flink,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs, # hue,mahout,giraph,crunch,pig,hive,zookeeper,ycsb,qfs # Example (to deploy only HDFS and YARN server and gateway parts) # This can be a comma-separated list or an array. @@ -191,3 +191,10 @@ zeppelin::server::spark_master_url: "yarn-client" zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}" zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}" zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}" + +# Flink +flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}" +flink::common::jobmanager_port: "6123" + +flink::common::ui_port: "8081" +flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}" diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index d4292697ab..b9ed6f98e2 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -15,6 +15,7 @@ hadoop::hadoop_storage_dirs: # - alluxio # - apex # - crunch +# - flink # - flume # - giraph # - ignite_hadoop diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index ba980bd23c..9ff424cdbe 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -56,6 +56,10 @@ master => ["alluxio-master"], worker => ["alluxio-worker"], }, + flink => { + master => ["flink-jobmanager"], + worker => ["flink-taskmanager"], + }, flume => { worker => ["flume-agent"], }, @@ -161,6 +165,7 @@ "alluxio", "apex", "crunch", + "flink", "giraph", "hadoop", "hadoop_hbase", diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp b/bigtop-deploy/puppet/modules/flink/manifests/init.pp new file mode 100644 index 0000000000..ae79fe45d5 --- /dev/null +++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp @@ -0,0 +1,62 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +class flink { + + class deploy ($roles) { + if ("flink-jobmanager" in $roles) { + include flink::jobmanager + } + + if ("flink-taskmanager" in $roles) { + include flink::taskmanager + } + } + + class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) { + # make sure flink is installed + package { "flink": + ensure => latest + } + + # set values in flink-conf.yaml + file { "/etc/flink/conf/flink-conf.yaml": + content => template("flink/flink-conf.yaml"), + require => Package["flink"] + } + } + + class jobmanager { + include common + + service { "flink-jobmanager": + ensure => running, + require => Package["flink"], + subscribe => File["/etc/flink/conf/flink-conf.yaml"], + hasrestart => true, + hasstatus => true + + } + + } + + class taskmanager { + include common + + service { "flink-taskmanager": + ensure => running, + require => Package["flink"], + subscribe => File["/etc/flink/conf/flink-conf.yaml"], + hasrestart => true, + hasstatus => true, + } + } +} diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml new file mode 100644 index 0000000000..14e8853bdf --- /dev/null +++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml @@ -0,0 +1,31 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + + +# Configuration values managed by puppet: +jobmanager.rpc.address: <%= @jobmanager_host %> +jobmanager.rpc.port: <%= @jobmanager_port %> +jobmanager.web.port: <%= @ui_port %> + +<% if defined?(storage_dirs) %> +taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %> +<% end %> + + +# For performance reasons its highly recommended to allocate as much memory to the +# Flink TaskManager as possible by setting 'taskmanager.heap.mb'. diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile index 63aff2014b..b4017d0ed7 100755 --- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile @@ -67,6 +67,7 @@ bigtop_master = "bigtop1.vagrant" $script = <