Skip to content
Browse files

adding a zookeeper demo cluster. Cleaned up some log statements.

  • Loading branch information...
1 parent 5bf67be commit 56769d227285eacefe22d8ec3f7ee821b9a15c2e Philip (flip) Kromer committed Jan 21, 2012
View
7 TODO.md
@@ -1,3 +1,10 @@
+* aws key databag + helper
+* cucumber
+* hbase
+* domain names
+* vagrants
+* launch machine with blank snapshot
+
### reorg
* symlinks in cookbooks
View
2 chefignore
@@ -38,4 +38,4 @@ a.out
.rspec
spec/*
spec/fixtures/*
-
+features/*
View
11 clusters/hbase_demo.rb
@@ -76,7 +76,16 @@ def hbase_facet(*components, &block)
[:nn, :jt, :nn2, :tt, :dn, :hm, :hm2, :rs, :hbth, :hbsg, :zk ]
- hbase_facet('master', :nn, :hm, :zk, :dn ){ instances 1 }
+ hbase_facet('master', :nn, :hm, :zk ) do
+ instances 1
+ # # The datanode and regionserver are only present for ease of bootstrapping
+ # # the machine
+ # role :hadoop_datanode
+ # role :hbase_regionserver
+ # facet_role.override_attributes({
+ # :hadoop => { :datanode => { :run_state => :stop, }, },
+ # :hbase => { :regionserver => { :run_state => :stop, }, }, })
+ end
hbase_facet('beta', :nn2, :hm2, :jt, :dn ){ instances 1 }
hbase_facet('worker', :rs, :tt, :dn ){ instances 3; role :hbase_stargate }
View
78 clusters/zookeeper_demo.rb
@@ -0,0 +1,78 @@
+#
+# A zookeeper quorum
+#
+# The only sensible numbers of instances to launch with are 3 or 5. A solo
+# zookeeper doesn't guarantee availability; and you should NEVER run an even
+# number of ZKs (http://hbase.apache.org/book/zookeeper.html).
+#
+ClusterChef.cluster 'zookeeper_demo' do
+ cloud(:ec2) do
+ defaults
+ availability_zones ['us-east-1d']
+ flavor 't1.micro'
+ backing 'ebs'
+ image_name 'cluster_chef-natty'
+ bootstrap_distro 'ubuntu10.04-cluster_chef'
+ mount_ephemerals(:tags => { :zookeeper_scratch => true, :zookeeper_data => false, })
+ end
+
+ # uncomment if you want to set your environment.
+ environment :prod
+
+ role :base_role
+ role :chef_client
+ role :ssh
+ role :nfs_client
+
+ role :volumes
+ role :package_set, :last
+ role :dashboard, :last
+
+ role :org_base
+ role :org_final, :last
+ role :org_users
+
+ role :tuning
+
+ facet :zookeeper do
+ instances 1
+ role :zookeeper_server
+ end
+
+ cluster_role.override_attributes({
+ })
+
+ # Launch the cluster with all of the below set to 'stop'.
+ #
+ # After initial bootstrap,
+ # * set the run_state to 'start' in the lines below
+ # * run `knife cluster sync bonobo-master` to push those values up to chef
+ # * run `knife cluster kick bonobo-master` to re-converge
+ #
+ # Once you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
+ # control panels, you're good to launch the rest of the cluster.
+ #
+ facet(:zookeeper).facet_role.override_attributes({
+ :zookeeper => {
+ :server => { :run_state => :start, },
+ },
+ })
+
+ #
+ # Attach 10GB persistent storage to each node, and use it for all zookeeper data_dirs.
+ #
+ # Modify the snapshot ID and attached volume size to suit
+ #
+ volume(:ebs1) do
+ defaults
+ size 10
+ keep true
+ device '/dev/sdk' # note: will appear as /dev/xvdi on natty
+ mount_point '/data/ebs1'
+ attachable :ebs
+ snapshot_id nil
+ tags( :zookeeper_data => true, :persistent => true, :local => false, :bulk => true, :fallback => false )
+ create_at_launch true # if no volume is tagged for that node, it will be created
+ end
+
+end
View
4 vendor/infochimps/hadoop_cluster/attributes/tunables.rb
@@ -90,7 +90,7 @@
if node[:memory][:swap] && node[:memory][:swap][:total]
ram -= node[:memory][:swap][:total].to_i
end
- Chef::Log.info("Couldn't set performance parameters from instance type, estimating from #{cores} cores and #{ram} ram")
+ Chef::Log.warn("Couldn't set performance parameters from instance type, estimating from #{cores} cores and #{ram} ram")
n_mappers = (cores >= 6 ? (cores * 1.25) : (cores * 2)).to_i
n_reducers = cores
heap_size = 0.75 * (ram.to_f / 1000) / (n_mappers + n_reducers)
@@ -101,7 +101,7 @@
{ :max_map_tasks => n_mappers, :max_reduce_tasks => n_reducers, :java_child_opts => "-Xmx#{heap_size}m", :java_child_ulimit => child_ulimit, :io_sort_factor => io_sort_factor, :io_sort_mb => io_sort_mb, }
end
-Chef::Log.info("Hadoop tunables: #{hadoop_performance_settings.inspect}")
+Chef::Log.debug("Hadoop tunables: #{hadoop_performance_settings.inspect}")
# (Mappers+Reducers)*ChildTaskHeap + DNheap + TTheap + 3GB + RSheap + OtherServices'
View
2 vendor/infochimps/volumes/definitions/volume_dirs.rb
@@ -60,8 +60,6 @@
params[:owner] ||= component.node_attr(:user, :required)
params[:group] ||= component.node_attr(:group) || params[:owner]
- Log.info( [params[:name], params, component.to_hash] )
-
#
# Once we've chosen a path, we need to use it forever.
#

0 comments on commit 56769d2

Please sign in to comment.
Something went wrong with that request. Please try again.