Merging Bigtop 0.2.0

git-svn-id: https://svn.apache.org/repos/asf/incubator/bigtop/branches/hadoop-0.23@1196308 13f79535-47bb-0310-9956-ffa450edef68

README

@@ -49,6 +49,21 @@ On Debian-based systems one also needs
 * build-essential dh-make debhelper devscripts
 * reprepro
 
+On openSUSE 11.4+, in addition, one also needs to ensure the following is installed:
+
+
+relaxngDatatype
+docbook-utils docbook-simple
+asciidoc
+fuse-devel
+docbook5
+docbook5-xsl-stylesheets
+libxml2-devel
+xmlformat
+xmlto
+libxslt
+libopenssl-devel
+
 ## Building packages
 
     $ make [component-name]-[rpm|deb]

bigtop-deploy/puppet/manifests/cluster.pp

@@ -0,0 +1,161 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop_cluster_node {
+  $hadoop_namenode_host="$hadoop_head_node"
+  $hadoop_namenode_port="17020"
+  $hadoop_namenode_thrift_port="10090"
+  # $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
+  # $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
+
+  $hadoop_jobtracker_host="$hadoop_head_node"
+  $hadoop_jobtracker_port="8021"
+  $hadoop_jobtracker_thrift_port="9290"
+  # $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
+  # $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
+
+  $hadoop_core_proxyusers = { oozie => { groups => 'root,hadoop,jenkins,oozie,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" } }
+
+  $hadoop_hbase_rootdir = "hdfs://$hadoop_namenode_host:$hadoop_namenode_port/hbase"
+  $hadoop_hbase_zookeeper_quorum = "$hadoop_head_node"
+
+  $hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
+
+  $namenode_data_dirs = ["/mnt/namenode"]
+  $hdfs_data_dirs = ["/mnt/hdfs"]
+  $mapred_data_dirs = ["/mnt/scratch"]
+
+  if ($hadoop_security_authentication == "kerberos") {
+    $kerberos_domain = "compute-1.internal"
+    $kerberos_realm = "EXAMPLE.COM"
+    $kerberos_kdc_server = "$hadoop_head_node"
+
+    include kerberos::client
+    kerberos::client::host_keytab { ["hdfs", "mapred", "hbase", "oozie"]:
+      princs_map => { hdfs   => [ "host", "hdfs" ],
+                      mapred => [ "mapred" ],
+                      hbase  => [ "hbase"  ],
+                      oozie  => [ "oozie"  ], },
+    }
+  }
+}
+
+class hadoop_worker_node inherits hadoop_cluster_node {
+  hadoop::datanode { "datanode":
+        namenode_host => $hadoop_namenode_host,
+        namenode_port => $hadoop_namenode_port,
+        dirs => $hdfs_data_dirs,
+        auth => $hadoop_security_authentication,
+  }
+
+  hadoop::tasktracker { "tasktracker":
+        namenode_host => $hadoop_namenode_host,
+        namenode_port => $hadoop_namenode_port,
+        jobtracker_host => $hadoop_jobtracker_host,
+        jobtracker_port => $hadoop_jobtracker_port,
+        dirs => $mapred_data_dirs,
+        auth => $hadoop_security_authentication,
+  }
+
+  hadoop-hbase::server { "hbase region server":
+        rootdir => $hadoop_hbase_rootdir,
+        zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
+        kerberos_realm => $kerberos_realm, 
+  }
+}
+
+class hadoop_head_node inherits hadoop_cluster_node {
+
+  if ($hadoop_security_authentication == "kerberos") {
+    include kerberos::kdc, kerberos::kdc::admin_server
+  }
+
+  hadoop::namenode { "namenode":
+        port => $hadoop_namenode_port,
+        jobtracker_host => $hadoop_jobtracker_host,
+        jobtracker_port => $hadoop_jobtracker_port,
+        dirs => $namenode_data_dirs,
+        # thrift_port => $hadoop_namenode_thrift_port,
+        auth => $hadoop_security_authentication,
+  }
+
+  hadoop::secondarynamenode { "secondary namenode":
+        namenode_host => $hadoop_namenode_host,
+        namenode_port => $hadoop_namenode_port,
+        auth => $hadoop_security_authentication,
+  }
+
+  hadoop::jobtracker { "jobtracker":
+        namenode_host => $hadoop_namenode_host,
+        namenode_port => $hadoop_namenode_port,
+        host => $hadoop_jobtracker_host,
+        port => $hadoop_jobtracker_port,
+        dirs => $mapred_data_dirs,
+        # thrift_port => $hadoop_jobtracker_thrift_port,
+        auth => $hadoop_security_authentication,
+  }
+
+  hadoop-hbase::master { "hbase master":
+        rootdir => $hadoop_hbase_rootdir,
+        zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
+        kerberos_realm => $kerberos_realm, 
+  }
+
+  hadoop-oozie::server { "oozie server":
+        kerberos_realm => $kerberos_realm, 
+  }
+
+  hadoop-zookeeper::server { "zookeeper":
+        myid => "0",
+        ensemble => $hadoop_zookeeper_ensemble,
+  }
+
+  hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user", "/hbase", "/benchmarks", "/user/jenkins", "/user/hive" ]:
+    hdfs_dirs_meta => { "/tmp"          => { perm => "777", user => "hdfs"   },
+                        "/mapred"       => { perm => "755", user => "mapred" },
+                        "/system"       => { perm => "755", user => "hdfs"   },
+                        "/user"         => { perm => "755", user => "hdfs"   },
+                        "/hbase"        => { perm => "755", user => "hbase"  },
+                        "/benchmarks"   => { perm => "777", user => "hdfs"   },
+                        "/user/jenkins" => { perm => "777", user => "jenkins"},
+                        "/user/hive"    => { perm => "777", user => "hive"   } },
+  }
+}
+
+class hadoop_gateway_node inherits hadoop_head_node {
+  # hadoop::client { "gateway":
+  #   namenode_host => $hadoop_namenode_host,
+  #   namenode_port => $hadoop_namenode_port,
+  #   jobtracker_host => $hadoop_jobtracker_host,
+  #   jobtracker_port => $hadoop_jobtracker_port,
+  #   # auth => $hadoop_security_authentication,
+  # }
+
+  hadoop-pig::client { "pig client":
+  }
+  hadoop-hive::client { "hive client":
+     hbase_zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
+  }
+  hadoop-sqoop::client { "sqoop client":
+  }
+  hadoop-oozie::client { "oozie client":
+  }
+  hadoop-hbase::client { "hbase client":
+  }
+  hadoop-zookeeper::client { "zookeeper client":
+  }
+  hadoop-flume::client { "flume client":
+  }
+}

bigtop-deploy/puppet/manifests/site.pp

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stage {"pre": before => Stage["main"]}
+
+yumrepo { "Bigtop":
+    baseurl => "http://bigtop01.cloudera.org:8080/job/Bigtop-trunk-centos5/lastSuccessfulBuild/artifact/output/",
+    descr => "Bigtop packages",
+    enabled => 1,
+    gpgcheck => 0,
+}
+
+package { "jdk":
+   ensure => "installed",
+}
+
+# $hadoop_head_node="beefy.my.org"
+# $hadoop_security_authentication="kerberos"
+
+import "cluster.pp"
+
+node default {
+  if ($hadoop_head_node == $fqdn) {
+    include hadoop_head_node
+  } else {
+    include hadoop_worker_node
+  }
+}

bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-flume {
+  define client {
+    package { "flume":
+      ensure => latest,
+    } 
+  }
+
+  # It really is flume node, but node is a reserved keyword for puppet
+  define agent {
+    package { "flume-node":
+      ensure => latest,
+    } 
+
+    service { "flume-node":
+      ensure => running,
+      require => Package["flume-node"],
+      # FIXME: this need to be fixed in upstream flume
+      hasstatus => false,
+      hasrestart => true,
+    }
+  }
+
+  define master {
+    package { "flume-master":
+      ensure => latest,
+    } 
+
+    service { "flume-master":
+      ensure => running,
+      require => Package["flume-node"],
+      # FIXME: this need to be fixed in upstream flume
+      hasstatus => true,
+      hasrestart => true,
+    }
+  }
+}

bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-flume::client { "test-flume": } 
+hadoop-flume::agent  { "test-flume-node": } 
+hadoop-flume::master { "test-flume-master": }
+
+

bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-hbase {
+  class client-package  {
+    package { "hadoop-hbase":
+      ensure => latest,
+    } 
+  }
+
+  class common-server-config {
+    include client-package
+
+    file { "/etc/hbase/conf/hbase-site.xml":
+      content => template("hadoop-hbase/hbase-site.xml"),
+      require => Package["hadoop-hbase"],
+    }
+    file { "/etc/hbase/conf/hbase-env.sh":
+      content => template("hadoop-hbase/hbase-env.sh"),
+      require => Package["hadoop-hbase"],
+    }
+  }
+
+  define client {
+    include client-package
+  }
+
+  define server($rootdir, $zookeeper_quorum, $kerberos_realm = "") {
+    include common-server-config
+
+    package { "hadoop-hbase-regionserver":
+      ensure => latest,
+    }
+
+    service { "hadoop-hbase-regionserver":
+      ensure => running,
+      require => Package["hadoop-hbase-regionserver"],
+      subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"],
+      hasrestart => true,
+      hasstatus => true,
+    } 
+  }
+
+  define master($rootdir, $zookeeper_quorum, $kerberos_realm = "") {
+    include common-server-config
+
+    package { "hadoop-hbase-master":
+      ensure => latest,
+    }
+
+    service { "hadoop-hbase-master":
+      ensure => running,
+      require => Package["hadoop-hbase-master"],
+      subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"],
+      hasrestart => true,
+      hasstatus => true,
+    } 
+  }
+}