lib/errors.rb @@ -17,7 +17,7 @@ namespace :gem do end task :install => :build do - `sudo gem install pkg/mandy-*.gem` + `sudo gem install pkg/mandy-*.gem --no-rdoc --no-ri` end task :push => :build do Rakefile @@ -56,7 +56,7 @@ puts "Packaging Gems for distribution..." payload = Mandy::Packer.pack(file, options.payload || ARGV[0], gemfile(options.gemfile)) cmdenv = options.cmdenv -at_exit do +at_exit do puts "Cleaning up..." Mandy::Packer.cleanup!(payload) puts "All done!" @@ -66,32 +66,35 @@ puts "Loading Mandy scripts..." require absolute_path(file) output = nil - -puts "Sending jobs to Hadoop..." - -Mandy::Job.jobs.each_with_index do |job, i| - - jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ') - output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}") +begin + Mandy::Job.jobs.each_with_index do |job, i| + puts "Running Job [#{i+1}] #{job.name}..." + + jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ') + output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}") - bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb')) - inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=</#{job.input_format_options[:xml_tag]}>" : nil + bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb')) + inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=</#{job.input_format_options[:xml_tag]}>" : nil - command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\ - -files "#{payload}","#{bootstrap_file}" \ - -conf '#{config}' \ - -input "#{input}" \ - #{ inputreader.nil? ? '' : "-inputreader \"#{inputreader}\"" } \ - -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'" \ - -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'" \ - #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\ - -output "#{output}") + command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\ + -files "#{payload}","#{bootstrap_file}" \ + -conf '#{config}' \ + -input "#{input}" \ + #{ inputreader.nil? ? '' : "-inputreader \"#{inputreader}\"" } \ + -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'" \ + -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'" \ + #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\ + -output "#{output}" 2>&1) - `#{command}` + result = `#{command}` + raise(Mandy::HadoopJobFailure.new(job, result)) unless $?.to_i==0 - # puts "#{command}" - input = output -end - -# print out the output location so caller can know where to get the results from -puts output \ No newline at end of file + # puts "#{command}" + input = output + end + # print out the output location so caller can know where to get the results from + puts output +rescue Mandy::HadoopJobFailure => e + STDERR.puts e.to_s + exit(1) +end \ No newline at end of file bin/mandy-hadoop @@ -26,6 +26,7 @@ require "cgi" stores/in_memory test_runner ruby-hbase + errors ).each {|file| require File.join(File.dirname(__FILE__), file) } module Mandy lib/mandy.rb @@ -7,8 +7,8 @@ Gem::Specification.new do |s| s.version = File.read(File.join(File.dirname(__FILE__),'VERSION')) s.date = '2009-10-19' - s.description = "Map/Reduce" - s.summary = "Map/Reduce" + s.description = "Mandy is Ruby Map/Reduce Framework built onto of the Hadoop Distributed computing platform." + s.summary = "Map/Reduce Framework" s.authors = ["Andy Kent", "Paul Ingles"] s.email = "andy.kent@me.com" @@ -53,6 +53,7 @@ Gem::Specification.new do |s| Rakefile bootstrap.rb geminstaller.yml + lib/errors.rb lib/mandy.rb lib/support/tuple.rb lib/support/formatting.rb mandy.gemspec 498c0292184c21555d7bc19fc2955b91060c2f5c Andy Kent andrew.d.kent@gmail.com http://github.com/trafficbroker/mandy/commit/02b3dfbeb681721745616bc0449fa862ec22fc9d 02b3dfbeb681721745616bc0449fa862ec22fc9d 2009-10-21T03:23:18-07:00 2009-10-21T03:23:18-07:00 much better hadoop error handling, we now return a non zero status code from the command line and print the important info to STDERR a3b7e5629e9ea0f826eb0f41a5a642f601eeff99 Andy Kent andrew.d.kent@gmail.com