Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

made the functionally-equivalent clusters look alike (and have their …

…differences line up next to each other)
  • Loading branch information...
commit 8f04323d8f109a1b9389a9c091a34b9d7ba67f8e 1 parent 6639d46
Philip (flip) Kromer authored
View
1  .gitignore
@@ -64,4 +64,5 @@ knife/plugins
.chef
vendor/ironfan-enterprise
vagrants/*
+.rbenv-version
Gemfile.lock
View
61 clusters/big_hadoop.rb
@@ -1,19 +1,30 @@
#
# Production cluster -- no persistent HDFS
#
+# !!Important setup steps!!:
+#
+# Launch the cluster with the hadoop daemon run states set to 'stop' -- see the
+# section most of the way down the page.
+#
+# After initial bootstrap,
+# * set the run_state to :start in the lines below
+# * run `knife cluster sync` to push those values up to chef
+# * run `knife cluster kick` to re-converge
+#
+# As soon as you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
+# control panels, you're good to launch the rest of the cluster.
+#
Ironfan.cluster 'big_hadoop' do
cloud(:ec2) do
defaults
+ permanent false
availability_zones ['us-east-1d']
flavor 'm1.large'
backing 'ebs'
image_name 'ironfan-natty'
bootstrap_distro 'ubuntu10.04-ironfan'
chef_client_script 'client.rb'
- mount_ephemerals(:tags => {
- :hadoop_scratch => true,
- :hadoop_data => true,
- })
+ mount_ephemerals(:tags => { :hadoop_scratch => true, :hadoop_data => true, :persistent => false, :bulk => true })
end
environment :dev
@@ -32,23 +43,25 @@
role :org_users
role :org_final, :last
- role :tuning
- role :jruby
- role :pig
-
role :hadoop
role :hadoop_s3_keys
- role :hbase_client
recipe 'hadoop_cluster::config_files', :last
- recipe 'hbase::config_files', :last
+ role :zookeeper_client, :last
+ role :hbase_client, :last
+
+ role :jruby
+ role :pig
+ recipe :rstats
+
+ role :tuning, :last
facet :master do
instances 1
role :hadoop_namenode
- role :hadoop_jobtracker
role :hadoop_secondarynn
- role :hadoop_tasktracker
+ role :hadoop_jobtracker
role :hadoop_datanode
+ role :hadoop_tasktracker
end
facet :worker do
@@ -59,29 +72,25 @@
cluster_role.override_attributes({
:hadoop => {
- :tasktracker => { :java_heap_size_max => 1400, },
+ # more jobtracker heap size for running large-mapper-count jobs
+ :jobtracker => { :java_heap_size_max => 2000, },
+ # lets you rapidly decommission nodes for elasticity
+ :balancer => { :max_bandwidth => (50 * 1024 * 1024) },
# make mid-flight data much smaller -- useful esp. with ec2 network constraints
:compress_mapout_codec => 'org.apache.hadoop.io.compress.SnappyCodec',
},
})
- # Launch the cluster with all of the below set to 'stop'.
- #
- # After initial bootstrap,
- # * set the run_state to :start in the lines below
- # * run `knife cluster sync hadoop_demo-master` to push those values up to chef
- # * run `knife cluster kick hadoop_demo-master` to re-converge
#
- # Once you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
- # control panels, you're good to launch the rest of the cluster.
+ # Hadoop Daemon run states
#
facet(:master).facet_role.override_attributes({
:hadoop => {
- :namenode => { :run_state => :stop, },
- :secondarynn => { :run_state => :stop, },
- :jobtracker => { :run_state => :stop, },
- :datanode => { :run_state => :stop, },
- :tasktracker => { :run_state => :stop, },
+ :namenode => { :run_state => :stop, },
+ :secondarynn => { :run_state => :stop, },
+ :jobtracker => { :run_state => :stop, },
+ :datanode => { :run_state => :stop, },
+ :tasktracker => { :run_state => :stop, },
},
})
View
32 clusters/control.rb
@@ -1,7 +1,10 @@
+#
+# Command and control cluster
+#
Ironfan.cluster 'control' do
-
cloud(:ec2) do
defaults
+ permanent true
availability_zones ['us-east-1d']
flavor 't1.micro'
backing 'ebs'
@@ -19,32 +22,33 @@
role :set_hostname
role :volumes
- role :package_set, :last
- role :minidash, :last
- role :tuning, :last
+ role :package_set, :last
+ role :minidash, :last
role :org_base
role :org_users
- role :org_final, :last
+ role :org_final, :last
facet :nfs do
role :nfs_server
+
facet_role do
override_attributes({
- :nfs => { :exports => { '/home' => { :name => 'home', :nfs_options => '*.internal(rw,no_root_squash,no_subtree_check)' }}},
+ :nfs => { :exports => {
+ '/home' => { :name => 'home', :nfs_options => '*.internal(rw,no_root_squash,no_subtree_check)' }}},
})
end
volume(:home_vol) do
defaults
- size 20
- keep true
- device '/dev/sdh' # note: will appear as /dev/xvdh on modern ubuntus
- mount_point '/home'
- attachable :ebs
- snapshot_name :blank_xfs
- resizable true
- create_at_launch true
+ size 20
+ keep true
+ device '/dev/sdh' # note: will appear as /dev/xvdh on modern ubuntus
+ mount_point '/home'
+ attachable :ebs
+ snapshot_name :blank_xfs
+ resizable true
+ create_at_launch true
tags( :persistent => true, :local => false, :bulk => false, :fallback => false )
end
end
View
122 clusters/el_ridiculoso.rb
@@ -23,94 +23,98 @@
role :chef_client
role :ssh
role :nfs_client
+ role :set_hostname
role :volumes
- role :package_set
- role :minidash, :last
- # role :set_hostname
+ role :package_set, :last
+ role :minidash, :last
role :org_base
- role :org_final, :last
role :org_users
+ role :org_final, :last
role :hadoop
role :hadoop_s3_keys
recipe 'hadoop_cluster::config_files', :last
- role :tuning, :last
- recipe 'cloud_utils::pickle_node'
+ role :hbase_client, :last
+ role :zookeeper_client, :last
+
+ # role :log_integration
+ # role :zabbix_agent, :last
+ recipe 'cloud_utils::pickle_node', :last
module ElRidiculoso
module_function
def master_processes
- role :cassandra_server
- role :elasticsearch_datanode
- role :elasticsearch_httpnode
- role :flume_master
- role :ganglia_master
- role :graphite_server
- role :hadoop_jobtracker
- role :hadoop_namenode
- role :hadoop_secondarynn
- role :hbase_master
+ role :cassandra_server
+ role :elasticsearch_datanode
+ role :elasticsearch_httpnode
+ role :flume_master
+ role :ganglia_master
+ role :graphite_server
+ role :hadoop_jobtracker
+ role :hadoop_namenode
+ role :hadoop_secondarynn
+ role :hbase_master
role :jenkins_server
- role :mongodb_server
- role :mysql_server
- role :redis_server
- role :resque_server
- role :statsd_server
+ role :mongodb_server
+ role :mysql_server
+ role :redis_server
+ role :resque_server
+ role :statsd_server
role :zabbix_server
role :zabbix_web
- role :zookeeper_server
+ role :zookeeper_server
# The default recipes for these run stuff even though it's impolite
recipe 'apache2'
recipe 'nginx'
end
def worker_processes
- role :flume_agent
- role :ganglia_agent
- role :hadoop_datanode
- role :hadoop_tasktracker
- role :hbase_regionserver
- role :hbase_stargate
- role :hbase_thrift
+ role :flume_agent
+ role :ganglia_agent
+ role :hadoop_datanode
+ role :hadoop_tasktracker
+ role :hbase_regionserver
+ role :hbase_stargate
+ role :hbase_thrift
role :jenkins_worker
end
def client_processes
- role :cassandra_client
- role :elasticsearch_client
- role :hbase_client
- role :mysql_client
- role :nfs_client
- role :redis_client
- role :zookeeper_client
+ role :cassandra_client
+ role :elasticsearch_client
+ role :hbase_client
+ role :mysql_client
+ role :nfs_client
+ role :redis_client
+ role :zookeeper_client
end
def simple_installs
- role :jruby
- role :pig
- recipe 'ant'
- recipe 'bluepill'
- recipe 'boost'
- recipe 'build-essential'
- recipe 'cron'
- recipe 'git'
- recipe 'hive'
- recipe 'java::sun'
- recipe 'jpackage'
- recipe 'jruby'
- recipe 'nodejs'
- recipe 'ntp'
- recipe 'openssh'
- recipe 'openssl'
- recipe 'rstats'
- recipe 'runit'
- recipe 'thrift'
- recipe 'xfs'
- recipe 'xml'
- recipe 'zabbix'
- recipe 'zlib'
+ role :jruby
+ role :pig
+ recipe 'ant'
+ recipe 'bluepill'
+ recipe 'boost'
+ recipe 'build-essential'
+ recipe 'cron'
+ recipe 'git'
+ recipe 'hive'
+ recipe 'java::sun'
+ recipe 'jpackage'
+ recipe 'jruby'
+ recipe 'nodejs'
+ recipe 'ntp'
+ recipe 'openssh'
+ recipe 'openssl'
+ recipe 'rstats'
+ recipe 'runit'
+ recipe 'thrift'
+ recipe 'xfs'
+ recipe 'xml'
+ recipe 'zabbix'
+ recipe 'zlib'
end
end
View
62 clusters/elastic_hadoop.rb
@@ -1,16 +1,30 @@
#
# Science cluster -- persistent HDFS
#
+# !!Important setup steps!!:
+#
+# Launch the cluster with the hadoop daemon run states set to 'stop' -- see the
+# section most of the way down the page.
+#
+# After initial bootstrap,
+# * set the run_state to :start in the lines below
+# * run `knife cluster sync` to push those values up to chef
+# * run `knife cluster kick` to re-converge
+#
+# As soon as you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
+# control panels, you're good to launch the rest of the cluster.
+#
Ironfan.cluster 'elastic_hadoop' do
cloud(:ec2) do
defaults
+ permanent false
availability_zones ['us-east-1d']
flavor 'm1.large'
backing 'ebs'
image_name 'ironfan-natty'
bootstrap_distro 'ubuntu10.04-ironfan'
chef_client_script 'client.rb'
- mount_ephemerals(:tags => { :hadoop_scratch => true, })
+ mount_ephemerals(:tags => { :hadoop_scratch => true, :hadoop_data => false, :persistent => false, :bulk => true })
end
environment :dev
@@ -29,23 +43,24 @@
role :org_users
role :org_final, :last
- role :tuning
- role :jruby
- role :pig
-
role :hadoop
role :hadoop_s3_keys
- role :hbase_client
recipe 'hadoop_cluster::config_files', :last
- recipe 'hbase::config_files', :last
+ role :zookeeper_client, :last
+ role :hbase_client, :last
+
+ role :jruby
+ role :pig
+
+ role :tuning, :last
facet :master do
instances 1
role :hadoop_namenode
- role :hadoop_jobtracker
role :hadoop_secondarynn
- role :hadoop_tasktracker
+ role :hadoop_jobtracker
role :hadoop_datanode
+ role :hadoop_tasktracker
end
facet :worker do
@@ -56,7 +71,8 @@
cluster_role.override_attributes({
:hadoop => {
- :tasktracker => { :java_heap_size_max => 1400, },
+ # more jobtracker heap size for running large-mapper-count jobs
+ :jobtracker => { :java_heap_size_max => 2000, },
# lets you rapidly decommission nodes for elasticity
:balancer => { :max_bandwidth => (50 * 1024 * 1024) },
# make mid-flight data much smaller -- useful esp. with ec2 network constraints
@@ -64,32 +80,22 @@
},
})
- # Launch the cluster with all of the below set to 'stop'.
- #
- # After initial bootstrap,
- # * set the run_state to :start in the lines below
- # * run `knife cluster sync hadoop_demo-master` to push those values up to chef
- # * run `knife cluster kick hadoop_demo-master` to re-converge
#
- # Once you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
- # control panels, you're good to launch the rest of the cluster.
+ # Hadoop Daemon run states
#
facet(:master).facet_role.override_attributes({
:hadoop => {
- :namenode => { :run_state => :stop, },
- :secondarynn => { :run_state => :stop, },
- :jobtracker => { :run_state => :stop, },
- :datanode => { :run_state => :stop, },
- :tasktracker => { :run_state => :stop, },
+ :namenode => { :run_state => :stop, },
+ :secondarynn => { :run_state => :stop, },
+ :jobtracker => { :run_state => :stop, },
+ :datanode => { :run_state => :stop, },
+ :tasktracker => { :run_state => :stop, },
},
})
- #
- # Attach persistent storage to each node, and use it for all hadoop data_dirs.
- #
volume(:ebs1) do
defaults
- size 200
+ size 10
keep true
device '/dev/sdj' # note: will appear as /dev/xvdj on modern ubuntus
mount_point '/data/ebs1'
@@ -97,7 +103,7 @@
snapshot_name :blank_xfs
resizable true
create_at_launch true
- tags( :hadoop_data => true, :zookeeper_data => true, :persistent => true, :local => false, :bulk => true, :fallback => false )
+ tags( :hadoop_data => true, :persistent => true, :local => false, :bulk => true, :fallback => false )
end
end
View
24 clusters/es.rb
@@ -1,6 +1,7 @@
Ironfan.cluster 'es' do
cloud(:ec2) do
defaults
+ # permanent true
availability_zones ['us-east-1d']
flavor 't1.micro'
backing 'ebs'
@@ -19,21 +20,34 @@
role :set_hostname
role :volumes
- role :package_set, :last
- role :minidash, :last
+ role :package_set, :last
+ role :minidash, :last
role :org_base
role :org_users
- role :org_final, :last
+ role :org_final, :last
+
+ role :tuning, :last
facet :elasticsearch do
- instances 1
+ num_nodes = 3
+ instances num_nodes
recipe 'volumes::build_raid', :first
- recipe 'tuning'
#
role :elasticsearch_server
+ role :elasticsearch_client
+
+ facet_role.override_attributes({
+ :elasticsearch => {
+ :expected_nodes => num_nodes,
+ :recovery_after_nodes => num_nodes,
+ :s3_gateway_bucket => "elasticsearch.#{Chef::Config[:organization]}.chimpy.us",
+ :server => { :run_state => :start }
+ }
+ })
raid_group(:md0) do
+ defaults
device '/dev/md0'
mount_point '/raid0'
level 0
View
148 clusters/hb.rb
@@ -1,26 +1,28 @@
-# Manually set all of the service states (see cluster_role below) to 'stop'.
#
-# Launch the namenode machine, allow it to converge, and then on that machine
-# run `/etc/hadoop/boostrap_hadoop_namenode.sh`.
+# HBase with persisten EBS-backed storage.
#
-# After initial bootstrap,
-# * set the run_state to :start in the lines below
-# * run `knife cluster sync bonobo-master` to push those values up to chef
-# * run `knife cluster kick bonobo-master` to re-converge
-#
-# Once you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
-# control panels, you're good to launch the rest of the cluster.
+# For serious use, you will want to go to larger nodes (`m1.xlarge` works well
+# for us) and should NOT use EBS-backed storage. We assume that for initial
+# experimentation you'll want to start/stop this, so it comes out of the box
+# with EBS.
#
-
+# !!Important setup steps!!:
#
+# Launch the cluster with the hadoop daemon run states set to 'stop' -- see the
+# section most of the way down the page.
#
-# * persistent HDFS --
+# After initial bootstrap,
+# * set the run_state to :start for all but the jobtracker and tasktracker (for an hbase client, these will typically be left at `:stop`)
+# * run `knife cluster sync` to push those values up to chef
+# * run `knife cluster kick` to re-converge
#
-# if you're testing, these recipes *will* work on a t1.micro. just don't use it for anything.
+# As soon as you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
+# control panels, you're good to launch the rest of the cluster.
#
Ironfan.cluster 'hb' do
cloud(:ec2) do
defaults
+ # permanent true
availability_zones ['us-east-1d']
flavor 'm1.large'
backing 'ebs'
@@ -30,8 +32,59 @@
mount_ephemerals(:tags => { :hbase_scratch => true, :hadoop_scratch => true })
end
+ flume_cluster_name = :dds
+ hbase_cluster_name = :hb
+ science_cluster_name = :elastic_hadoop
+ zookeeper_cluster_name = :zk
+
environment :dev
+ cluster_overrides = Mash.new({
+ # Look for the zookeeper nodes in the dedicated zookeeper cluster
+ :discovers => {
+ :zookeeper => { :server => zookeeper_cluster_name },
+ },
+ :hadoop => {
+ :namenode => { :run_state => :start, },
+ :secondarynn => { :run_state => :start, },
+ :datanode => { :run_state => :start, },
+ :jobtracker => { :run_state => :stop, }, # leave this at 'stop', usually
+ :tasktracker => { :run_state => :stop, }, # leave this at 'stop', usually
+ :compress_mapout_codec => 'org.apache.hadoop.io.compress.SnappyCodec',
+ },
+ :hbase => {
+ :master => { :run_state => :start },
+ :regionserver => { :run_state => :start },
+ :stargate => { :run_state => :start }, },
+ :zookeeper => {
+ :server => { :run_state => :start }, },
+ })
+
+ #
+ ## Uncomment the lines below to stop all services on the cluster
+ #
+ # cluster_overrides[:hadoop ][:namenode ][:run_state] = :stop
+ # cluster_overrides[:hadoop ][:secondarynn ][:run_state] = :stop
+ # cluster_overrides[:hadoop ][:datanode ][:run_state] = :stop
+ # cluster_overrides[:hbase ][:master ][:run_state] = :stop
+ # cluster_overrides[:hbase ][:regionserver ][:run_state] = :stop
+ # cluster_overrides[:hbase ][:stargate ][:run_state] = :stop
+ # cluster_overrides[:zookeeper][:server ][:run_state] = :stop
+
+ # # total size of the JVM heap (regionserver) (default 2000m)
+ # cluster_overrides[:hbase][:regionserver][:java_heap_size_max] = "4000m"
+
+ # # hbase.hregion.memstore.mslab.enabled (default false) -- Experimental: Enables the
+ # # MemStore-Local Allocation Buffer, a feature which works to prevent heap fragmentation
+ # # under heavy write loads. This can reduce the frequency of stop-the-world GC pauses on
+ # # large heaps.
+ # cluster_overrides[:hbase][:memstore] ||= {}
+ # cluster_overrides[:hbase][:memstore][:mslab_enabled] = true
+
+ # # Setting this to 0 entirely removes the limit on concurrent connections. This is necessary
+ # # to overcome https://issues.apache.org/jira/browse/HBASE-4684 in HBase 0.90.4
+ # cluster_overrides[:zookeeper][:max_client_connections] = 0
+
role :systemwide
role :chef_client
role :ssh
@@ -44,68 +97,54 @@
role :org_base
role :org_users
- role :org_final, :last
-
- role :tuning
- role :jruby
- role :pig
+ role :org_final, :last
role :hadoop
role :hadoop_s3_keys
recipe 'hadoop_cluster::config_files', :last
+ role :zookeeper_client, :last
+ role :hbase_client, :last
recipe 'hbase::config_files', :last
+ role :jruby
+ role :pig
+
+ role :tuning, :last
+
facet :alpha do
- instances 1
- role :hadoop_namenode
- role :hbase_master
+ instances 1
+ role :hadoop_namenode
+ role :hbase_master
end
facet :beta do
- instances 1
- role :hadoop_secondarynn
- role :hadoop_jobtracker
- role :hbase_master
+ instances 1
+ role :hadoop_secondarynn
+ role :hadoop_jobtracker
+ role :hbase_master
end
facet :worker do
- instances 4
- role :hadoop_datanode
- role :hadoop_tasktracker
- role :hbase_regionserver
- role :hbase_stargate
- role :hbase_thrift
+ instances 3
+ role :hadoop_datanode
+ role :hadoop_tasktracker
+ role :hbase_regionserver
+ role :hbase_stargate
+ role :hbase_thrift
end
# This line, and the 'discovers' setting in the cluster_role,
# enable the hbase to use an external zookeeper cluster
- self.cloud.security_group(self.name).authorized_by_group("zk")
-
- cluster_role.override_attributes({
- # Look for the zookeeper nodes in the dedicated zookeeper cluster
- :discovers => {
- :zookeeper => { :server => 'zk' }, },
- #
- :hadoop => {
- :namenode => { :run_state => :start, },
- :secondarynn => { :run_state => :start, },
- :datanode => { :run_state => :start, },
- :jobtracker => { :run_state => :stop, },
- :tasktracker => { :run_state => :stop, },
- :compress_mapout_codec => 'org.apache.hadoop.io.compress.SnappyCodec',
- },
- :hbase => {
- :master => { :run_state => :start },
- :regionserver => { :run_state => :start },
- :stargate => { :run_state => :start }, },
- :zookeeper => {
- :server => { :run_state => :stop }, },
- })
+ self.cloud.security_group(self.name) do
+ authorized_by_group(zookeeper_cluster_name)
+ authorize_group(flume_cluster_name)
+ authorize_group(science_cluster_name)
+ end
#
# Attach persistent storage to each node, and use it for all hadoop data_dirs.
#
volume(:ebs1) do
defaults
- size 200
+ size 10
keep true
device '/dev/sdj' # note: will appear as /dev/xvdj on modern ubuntus
mount_point '/data/ebs1'
@@ -116,4 +155,5 @@
tags( :hbase_data => true, :hadoop_data => true, :zookeeper_data => true, :persistent => true, :local => false, :bulk => true, :fallback => false )
end
+ cluster_role.override_attributes(cluster_overrides)
end
View
5 clusters/sandbox.rb
@@ -4,6 +4,7 @@
Ironfan.cluster 'sandbox' do
cloud(:ec2) do
defaults
+ permanent false
availability_zones ['us-east-1d']
flavor 't1.micro'
backing 'ebs'
@@ -29,6 +30,8 @@
role :org_users
role :org_final, :last
+ role :tuning, :last
+
facet :simple do
instances 1
end
@@ -44,7 +47,7 @@
device '/dev/md0'
mount_point '/raid0'
level 0
- sub_volumes [:ephemeral0, :ephemeral1] # , :ephemeral2, :ephemeral3]
+ sub_volumes [:ephemeral0, :ephemeral1]
end
end
View
8 clusters/web.rb
@@ -1,6 +1,7 @@
Ironfan.cluster 'web' do
cloud :ec2 do
defaults
+ # permanent true
availability_zones ['us-east-1d']
flavor 't1.micro' # change to something larger for serious use
backing 'ebs'
@@ -19,7 +20,12 @@
role :set_hostname
role :volumes
- role :package_set, :last
+ role :package_set, :last
+ role :minidash, :last
+
+ role :org_base
+ role :org_users
+ role :org_final, :last
facet :webnode do
instances 6
View
10 clusters/zk.rb
@@ -8,6 +8,7 @@
Ironfan.cluster 'zk' do
cloud(:ec2) do
defaults
+ permanent true
availability_zones ['us-east-1d']
flavor 't1.micro' # change to something larger for serious use
backing 'ebs'
@@ -26,16 +27,17 @@
role :set_hostname
role :volumes
- role :package_set, :last
- role :minidash, :last
+ role :package_set, :last
+ role :minidash, :last
role :org_base
role :org_users
- role :org_final, :last
+ role :org_final, :last
- role :tuning
role :jruby
+ role :tuning, :last
+
facet :zookeeper do
instances 1
role :zookeeper_server
2  vendor/ironfan-pantry
@@ -1 +1 @@
-Subproject commit 95036413f23559afa3aee79df24f7f15e689ae8d
+Subproject commit 0a2da935ef64aeb782c49ccd110dc06bf9f9d219

0 comments on commit 8f04323

Please sign in to comment.
Something went wrong with that request. Please try again.