WIP flink hacking --rpm build complete

bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml

@@ -124,6 +124,7 @@ hcatalog::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 hcatalog::webhcat::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 
 spark::common::master_host: "%{hiera('bigtop::hadoop_head_node')}"
+flink::common::master_name: "%{hiera('bigtop::hadoop_head_node')}"
 
 tachyon::common::master_host: "%{hiera('bigtop::hadoop_head_node')}"
 

bigtop-deploy/puppet/manifests/cluster.pp

@@ -89,6 +89,10 @@
   pig => {
     client => ["pig-client"],
   },
+  flink => {
+    master => ["flink-master"],
+    worker => ["flink-worker"],
+  },
   hive => {
     client => ["hive-client"],
   },
@@ -159,6 +163,7 @@
     "hadoop_hive",
     "hadoop_oozie",
     "hadoop_pig",
+    "flink",
     "sqoop2",
     "hadoop_zookeeper",
     "hcatalog",

bigtop-deploy/puppet/manifests/site.pp

@@ -66,6 +66,7 @@
   alias => "jdk",
 }
 
+
 import "cluster.pp"
 
 node default {

bigtop-deploy/puppet/modules/flink/manifests/init.pp

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class flink {
+
+  class deploy ($roles) {
+
+   if ('flink-master' in $roles) {
+      include flink::master
+    }
+
+    if ('flink-worker' in $roles) {
+      include flink::worker
+    }
+
+}
+
+class master {
+    include common   
+    service { "jobmanager":
+      ensure     => running,
+      require  => [
+        Package["flink"],
+        File["/etc/flink/conf/flink-env.sh"],
+	File["/etc/flink/conf/flink-conf.yaml"],
+	File["/etc/flink/conf/masters"]
+	],
+      hasrestart => true,
+      hasstatus  => true,
+    }
+}
+
+  class worker {
+    include common
+    service { "taskmanager":
+      ensure     => running,
+      require  => [
+        Package["flink"],
+        File["/etc/flink/conf/flink-env.sh"],
+        File["/etc/flink/conf/flink-conf.yaml"],
+ 	File["/etc/flink/conf/masters"]
+      ],
+      hasrestart => true,
+      hasstatus  => true,
+     }
+}
+
+
+  class common(
+      $master_host = $fqdn,
+      $master_port = 6123,
+      $master_ui_port = 8081,
+     )
+  {
+      package { "flink":
+      ensure => latest,
+    }
+
+
+  file { '/etc/flink/conf/flink-conf.yaml':
+	ensure => present,
+	content => template('flink/flink-conf.yaml'),
+	require => Package['flink'],
+	mode => 755,
+  }
+
+  file { '/etc/flink/conf/flink-env.sh':
+	ensure => present,
+	content => template('flink/flink-env.sh'),
+	require => Package['flink'],
+	mode => 755,
+  }
+  file { '/etc/flink/conf/masters':
+	ensure => present,
+	content => template('flink/masters'),
+	require => Package['flink'],
+	mode => 755,
+    }
+  }
+}         

bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml

@@ -0,0 +1,154 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+#==============================================================================
+# Common
+#==============================================================================
+
+# The host on which the JobManager runs. Only used in non-high-availability mode.
+# The JobManager process will use this hostname to bind the listening servers to.
+# The TaskManagers will try to connect to the JobManager on that host.
+
+jobmanager.rpc.address: bigtop1.vagrant
+
+
+# The port where the JobManager's main actor system listens for messages.
+
+jobmanager.rpc.port: 6123
+taskmanager.rpc.port: 6122
+
+# The heap size for the JobManager JVM
+
+jobmanager.heap.mb: 256
+
+
+# The heap size for the TaskManager JVM
+
+taskmanager.heap.mb: 512
+
+
+# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
+
+taskmanager.numberOfTaskSlots: 1
+
+# Specify whether TaskManager memory should be allocated when starting up (true) or when
+# memory is required in the memory manager (false)
+
+taskmanager.memory.preallocate: false
+
+# The parallelism used for programs that did not specify and other parallelism.
+
+parallelism.default: 1
+
+
+#==============================================================================
+# Web Frontend
+#==============================================================================
+
+# The port under which the web-based runtime monitor listens.
+# A value of -1 deactivates the web server.
+
+jobmanager.web.port: 8081
+
+# Flag to specify whether job submission is enabled from the web-based
+# runtime monitor. Uncomment to disable.
+
+#jobmanager.web.submit.enable: false
+
+
+#==============================================================================
+# Streaming state checkpointing
+#==============================================================================
+
+# The backend that will be used to store operator state checkpoints if
+# checkpointing is enabled.
+#
+# Supported backends: jobmanager, filesystem, <class-name-of-factory>
+#
+#state.backend: filesystem
+
+
+# Directory for storing checkpoints in a Flink-supported filesystem
+# Note: State backend must be accessible from the JobManager and all TaskManagers.
+# Use "hdfs://" for HDFS setups, "file://" for UNIX/POSIX-compliant file systems,
+# (or any local file system under Windows), or "S3://" for S3 file system.
+#
+# state.backend.fs.checkpointdir: hdfs://namenode-host:port/flink-checkpoints
+
+
+#==============================================================================
+# Advanced
+#==============================================================================
+
+# The number of buffers for the network stack.
+#
+# taskmanager.network.numberOfBuffers: 2048
+
+
+# Directories for temporary files.
+#
+# Add a delimited list for multiple directories, using the system directory
+# delimiter (colon ':' on unix) or a comma, e.g.:
+#     /data1/tmp:/data2/tmp:/data3/tmp
+#
+# Note: Each directory entry is read from and written to by a different I/O
+# thread. You can include the same directory multiple times in order to create
+# multiple I/O threads against that directory. This is for example relevant for
+# high-throughput RAIDs.
+#
+# If not specified, the system-specific Java temporary directory (java.io.tmpdir
+# property) is taken.
+#
+# taskmanager.tmp.dirs: /tmp
+
+
+# Path to the Hadoop configuration directory.
+#
+# This configuration is used when writing into HDFS. Unless specified otherwise,
+# HDFS file creation will use HDFS default settings with respect to block-size,
+# replication factor, etc.
+#
+# You can also directly specify the paths to hdfs-default.xml and hdfs-site.xml
+# via keys 'fs.hdfs.hdfsdefault' and 'fs.hdfs.hdfssite'.
+#
+# fs.hdfs.hadoopconf: /path/to/hadoop/conf/
+
+#==============================================================================
+# HDFS configuration
+#==============================================================================
+
+fs.hdfs.hdfsite : /etc/hadoop/conf/hdfs-site.xml
+fs.hdfs.hadoopconf : /etc/hadoop/conf 
+
+#==============================================================================
+# Master High Availability (required configuration)
+#==============================================================================
+
+# The list of ZooKepper quorum peers that coordinate the high-availability
+# setup. This must be a list of the form:
+# "host1:clientPort,host2[:clientPort],..." (default clientPort: 2181)
+#
+# recovery.mode: zookeeper
+#
+# recovery.zookeeper.quorum: localhost:2181,...
+#
+# Note: You need to set the state backend to 'filesystem' and the checkpoint
+# directory (see above) before configuring the storageDir.
+#
+# recovery.zookeeper.storageDir: hdfs:///recovery

bigtop-deploy/puppet/modules/flink/templates/flink-env.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export FLINK_HOME=${FLINK_HOME:-/usr/lib/flink}
+export FLINK_LOG_DIR=${FLINK_LOG_DIR:-/var/log/flink}
+export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+
+export FLINK_MASTER_HOST=bigtop1.vagrant
+export FLINK_MASTER_PORT=<%= @master_port %>
+export FLINK_MASTER_IP=$FLINK_MASTER_HOST
+export FLINK_MASTER_WEBUI_PORT=<%= @master_ui_port %>

bigtop-deploy/puppet/modules/flink/templates/masters

@@ -0,0 +1 @@
+bigtop1.vagrant

bigtop-deploy/vm/vagrant-puppet-openstack/vagrantconfig.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"
 num_instances: 1
 distro: centos
 components: [hadoop, yarn]

bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile

@@ -87,6 +87,9 @@ EOF
 SCRIPT
 
 VAGRANTFILE_API_VERSION = "2"
+
+slave_ip=[]
+
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager)
   # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
@@ -97,6 +100,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.cache.scope = :box
   end
 
+
   # nodes definition
   (1..num_instances).each do |i|
     config.vm.define "bigtop#{i}" do |bigtop|
@@ -105,6 +109,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       bigtop_hostname="bigtop#{i}"
       bigtop_fqdn="#{bigtop_hostname}.vagrant"
       bigtop_ip="10.10.10.1#{i}"
+      slave_ip.push(bigtop_ip)
 
       bigtop.vm.provider :virtualbox do |vb|
         vb.customize ["modifyvm", :id, "--memory", CONF['memory_size']]
@@ -136,6 +141,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         puppet.manifests_path =  "../../puppet/manifests/"
         puppet.manifest_file  = "site.pp"
         puppet.options = "--debug"
+        puppet.facter = {'allslaves' => slave_ip }
       end
 
       if run_smoke_tests then

bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml

@@ -17,10 +17,14 @@ memory_size: 4096
 number_cpus: 1
 box: "puppetlabs/centos-7.0-64-nocm"
 repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"
-num_instances: 1
+num_instances: 2 
 distro: centos
-components: [hadoop, yarn]
-enable_local_repo: false
+components: [hadoop, yarn, flink]
+<<<<<<< HEAD
+enable_local_repo: true 
+=======
+enable_local_repo: true
+>>>>>>> 64a00c45c9e21ac2ad39a334666b9dd5e21037ba
 run_smoke_tests: false
 smoke_test_components: [mapreduce, pig]
 jdk: "java-1.7.0-openjdk-devel.x86_64"

bigtop-packages/src/common/flink/do-component-build

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+
+#mvn clean install -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@"
+mvn clean install -Drat.skip=true -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@"