More cleanup of vendored examples into homebase

example_clusters/big_hadoop.rb

@@ -0,0 +1,99 @@
+#
+# Production cluster -- no persistent HDFS
+#
+# !!Important setup steps!!:
+#
+# Launch the cluster with the hadoop daemon run states set to 'stop' -- see the
+# section most of the way down the page.
+#
+# After initial bootstrap,
+# * set the run_state to :start in the lines below
+# * run `knife cluster sync` to push those values up to chef
+# * run `knife cluster kick` to re-converge
+#
+# As soon as you see 'nodes=1' on jobtracker (host:50030) & namenode (host:50070)
+# control panels, you're good to launch the rest of the cluster.
+#
+Ironfan.cluster 'big_hadoop' do
+  cloud(:ec2) do
+    permanent           false
+    availability_zones ['us-east-1d']
+    flavor              'm1.large'
+    backing             'ebs'
+    image_name          'ironfan-natty'
+    bootstrap_distro    'ubuntu10.04-ironfan'
+    chef_client_script  'client.rb'
+    mount_ephemerals(:tags => { :hadoop_scratch => true, :hadoop_data => true, :persistent => false, :bulk => true  })
+  end
+
+  environment           :dev
+
+  role                  :systemwide
+  cloud(:ec2).security_group :systemwide
+  role                  :chef_client
+  role                  :ssh
+  cloud(:ec2).security_group(:ssh).authorize_port_range 22..22
+  role                  :nfs_client
+  cloud(:ec2).security_group :nfs_client
+  role                  :set_hostname
+
+  role                  :volumes
+  role                  :package_set,   :last
+  role                  :minidash,      :last
+
+  role                  :org_base
+  role                  :org_users
+  role                  :org_final,     :last
+
+  role                  :hadoop
+  role                  :hadoop_s3_keys
+  recipe                'hadoop_cluster::config_files', :last
+  role                  :zookeeper_client, :last
+  role                  :hbase_client,     :last
+
+  role                  :jruby
+  role                  :pig
+  recipe                :rstats
+
+  role                  :tuning,        :last
+
+  facet :master do
+    instances           1
+    role                :hadoop_namenode
+    role                :hadoop_secondarynn
+    role                :hadoop_jobtracker
+    role                :hadoop_datanode
+    role                :hadoop_tasktracker
+  end
+
+  facet :worker do
+    instances           2
+    role                :hadoop_datanode
+    role                :hadoop_tasktracker
+  end
+
+  cluster_role.override_attributes({
+      :hadoop => {
+        # more jobtracker heap size for running large-mapper-count jobs
+        :jobtracker  => { :java_heap_size_max => 2000, },
+        # lets you rapidly decommission nodes for elasticity
+        :balancer    => { :max_bandwidth => (50 * 1024 * 1024) },
+        # make mid-flight data much smaller -- useful esp. with ec2 network constraints
+        :compress_mapout_codec => 'org.apache.hadoop.io.compress.SnappyCodec',
+      },
+    })
+
+  #
+  # Hadoop Daemon run states
+  #
+  facet(:master).facet_role.override_attributes({
+      :hadoop => {
+        :namenode     => { :run_state => :stop,  },
+        :secondarynn  => { :run_state => :stop,  },
+        :jobtracker   => { :run_state => :stop,  },
+        :datanode     => { :run_state => :stop,  },
+        :tasktracker  => { :run_state => :stop,  },
+      },
+    })
+
+end

example_clusters/burninator.rb

@@ -0,0 +1,85 @@
+#
+# Burninator cluster -- populate an AMI with installed software, but no
+# services, users or other preconceptions.
+#
+# The script /tmp/burn_ami_prep.sh will help finalize the machine -- then, just
+# stop it and invoke 'Create Image (EBS AMI)'.
+#
+Ironfan.cluster 'burninator' do
+  cloud(:ec2) do
+    availability_zones ['us-east-1d']
+    # use a c1.xlarge so the AMI knows about all ephemeral drives
+    flavor              'c1.xlarge'
+    backing             'ebs'
+    # image_name is per-facet here
+    bootstrap_distro    'ubuntu10.04-ironfan'
+    chef_client_script  'client.rb'
+    mount_ephemerals
+  end
+
+  environment           :dev
+
+  role                  :chef_client
+  role                  :ssh
+  cloud(:ec2).security_group(:ssh).authorize_port_range 22..22
+
+  # It's handy to have the root volumes not go away with the machine.
+  # It also means you can find yourself with a whole ton of stray 8GB
+  # images once you're done burninatin' so make sure to go back and
+  # clear them out
+  volume(:root).keep    true
+
+  #
+  # A throwaway facet for AMI generation
+  #
+  facet :trogdor do
+    instances           1
+
+    cloud(:ec2).image_name    'natty'  # Leave set at vanilla natty
+
+    recipe              'cloud_utils::burn_ami_prep'
+
+    role                :package_set, :last
+
+    recipe              'ant'
+    recipe              'boost'
+    recipe              'build-essential'
+    recipe              'emacs'
+    recipe              'git'
+    recipe              'java::sun'
+    recipe              'jpackage'
+    recipe              'jruby'
+    recipe              'jruby::gems'
+    recipe              'nodejs'
+    recipe              'ntp'
+    recipe              'openssl'
+    recipe              'pig::install_from_release'
+    recipe              'hadoop_cluster::add_cloudera_repo'
+    recipe              'runit'
+    recipe              'thrift'
+    recipe              'xfs'
+    recipe              'xml'
+    recipe              'zlib'
+    recipe              'zsh'
+
+    facet_role.override_attributes({
+        :java        => { :install_flavor => 'sun' }, # use sun java typically
+        :package_set => { :install => %w[ base dev sysadmin text python emacs ] },
+        :apt         => { :cloudera => { :force_distro => 'maverick',  }, },
+      })
+  end
+
+  #
+  # Used to test the generated AMI.
+  #
+  facet :village do
+    instances     1
+    # Once the AMI is burned, add a new entry in your knife configuration -- see
+    # knife/example-credentials/knife-org.rb. Fill in its name here:
+    cloud(:ec2).image_name    'ironfan-natty'
+
+    # just so there's something in the runlist.
+    recipe              'motd'
+  end
+
+end

example_clusters/control.rb

@@ -0,0 +1,57 @@
+#
+# Command and control cluster
+#
+Ironfan.cluster 'control' do
+  cloud(:ec2) do
+    permanent           true
+    availability_zones ['us-east-1d']
+    flavor              't1.micro'
+    backing             'ebs'
+    image_name          'ironfan-natty'
+    bootstrap_distro    'ubuntu10.04-ironfan'
+    chef_client_script  'client.rb'
+    mount_ephemerals
+  end
+
+  environment           :dev
+
+  role                  :systemwide
+  cloud(:ec2).security_group :systemwide
+  role                  :chef_client
+  role                  :ssh
+  cloud(:ec2).security_group(:ssh).authorize_port_range 22..22
+  role                  :set_hostname
+
+  role                  :volumes
+  role                  :package_set,   :last
+  role                  :minidash,      :last
+
+  role                  :org_base
+  role                  :org_users
+  role                  :org_final,     :last
+
+  facet :nfs do
+    role                :nfs_server
+    cloud(:ec2).security_group(:nfs_server).authorize_group :nfs_client
+
+    facet_role do
+      override_attributes({
+          :nfs => { :exports => {
+              '/home' => { :name => 'home', :nfs_options => '*.internal(rw,no_root_squash,no_subtree_check)' }}},
+        })
+    end
+
+    volume(:home_vol) do
+      size              20
+      keep              true
+      device            '/dev/sdh' # note: will appear as /dev/xvdh on modern ubuntus
+      mount_point       '/home'
+      attachable        :ebs
+      snapshot_name     :blank_xfs
+      resizable         true
+      create_at_launch  true
+      tags( :persistent => true, :local => false, :bulk => false, :fallback => false )
+    end
+  end
+
+end