Permalink
Browse files

Add complete local Hadoop setup as a Rake task

This will be needed to run in CI
  • Loading branch information...
1 parent 07f1591 commit e921273f27a3adb59c2807246ef9a3e9de96907b @iconara committed Aug 11, 2013
View
@@ -4,4 +4,4 @@
/pkg
/doc
/.yardoc
-/.classpath
+/spec/hadoop_setup.rb
View
@@ -0,0 +1 @@
+rubydoop
View
@@ -0,0 +1 @@
+jruby-1.7.4
View
3 .rvmrc
@@ -1,3 +0,0 @@
-hadoop classpath | tr : "\n" | sed 's,/bin/..,,' > .classpath
-rvm --create use jruby-1.6.7@rubydoop
-rvm rvmrc trust spec/integration/test_project
View
@@ -1,9 +1,12 @@
-source :rubygems
+source 'https://rubygems.org/'
group :development do
gem 'yard'
gem 'maruku'
gem 'pry'
+ gem 'travis'
+ gem 'json'
+ gem 'rake'
end
group :test do
View
@@ -1,16 +1,47 @@
GEM
- remote: http://rubygems.org/
+ remote: https://rubygems.org/
specs:
+ addressable (2.3.5)
+ backports (3.3.3)
coderay (1.0.7)
diff-lcs (1.1.3)
+ ethon (0.5.12)
+ ffi (>= 1.3.0)
+ mime-types (~> 1.18)
+ faraday (0.8.8)
+ multipart-post (~> 1.2.0)
+ faraday_middleware (0.9.0)
+ faraday (>= 0.7.4, < 0.9)
+ ffi (1.9.0-java)
+ gh (0.11.3)
+ addressable
+ backports
+ faraday (~> 0.8)
+ multi_json (~> 1.0)
+ net-http-persistent (>= 2.7)
+ net-http-pipeline
+ highline (1.6.19)
+ json (1.8.0-java)
+ launchy (2.3.0)
+ addressable (~> 2.3)
maruku (0.6.0)
syntax (>= 1.0.0)
method_source (0.8)
+ mime-types (1.23)
+ multi_json (1.7.8)
+ multipart-post (1.2.0)
+ net-http-persistent (2.9)
+ net-http-pipeline (1.0.1)
+ netrc (0.7.7)
pry (0.9.10-java)
coderay (~> 1.0.5)
method_source (~> 0.8)
slop (~> 3.3.1)
spoon (~> 0.0)
+ pusher-client (0.3.1)
+ ruby-hmac (~> 0.4.0)
+ websocket (~> 1.0.0)
+ rake (10.0.4)
rspec (2.11.0)
rspec-core (~> 2.11.0)
rspec-expectations (~> 2.11.0)
@@ -19,18 +50,38 @@ GEM
rspec-expectations (2.11.3)
diff-lcs (~> 1.1.3)
rspec-mocks (2.11.2)
+ ruby-hmac (0.4.0)
rvm (1.11.3.5)
slop (3.3.3)
spoon (0.0.1)
syntax (1.0.0)
+ travis (1.5.0)
+ backports
+ faraday (~> 0.8)
+ faraday_middleware (~> 0.9)
+ gh
+ highline (~> 1.6)
+ launchy (~> 2.1)
+ netrc (~> 0.7)
+ pry (~> 0.9)
+ pusher-client (~> 0.3, >= 0.3.1)
+ typhoeus (~> 0.5)
+ websocket-native (~> 1.0)
+ typhoeus (0.6.3)
+ ethon (~> 0.5.11)
+ websocket (1.0.7)
+ websocket-native (1.0.0-java)
yard (0.8.2.1)
PLATFORMS
java
DEPENDENCIES
+ json
maruku
pry
+ rake
rspec
rvm
+ travis
yard
View
@@ -1,10 +1,8 @@
# encoding: utf-8
require 'ant'
-require 'rspec/core/rake_task'
-
-namespace :build do
+namespace :dist do
source_dir = 'ext/src'
build_dir = 'ext/build'
ruby_dir = 'lib'
@@ -39,20 +37,58 @@ namespace :build do
end
desc 'Build the lib/rubydoop.jar'
-task :build => 'build:jars'
+task :dist => 'dist:jars'
-RSpec::Core::RakeTask.new(:spec)
+namespace :setup do
+ task :hadoop do
+ hadoop_release = ENV['HADOOP_RELEASE'] || 'hadoop-1.0.3/hadoop-1.0.3-bin'
+ hadoop_url = "http://archive.apache.org/dist/hadoop/common/#{hadoop_release}.tar.gz"
+ FileUtils.mkdir_p('tmp')
+ Dir.chdir('tmp') do
+ command = (<<-END).lines.map(&:strip).join(' && ')
+ rm -fr hadoop*
+ curl --progress-bar -O '#{hadoop_url}'
+ tar xf hadoop*.tar.gz
+ END
+ system(command)
+ end
+ end
-desc 'Tag & release the gem'
-task :release => %w[build:clean build spec] do
- $: << 'lib'
- require 'rubydoop/version'
+ task :test_project do
+ Dir.chdir('spec/integration/test_project') do
+ command = (<<-END).lines.map(&:strip).join(' && ')
+ rvm gemset create rubydoop-test_project
+ rvm $RUBY_VERSION@rubydoop-test_project do bundle install
+ END
+ puts command
+ Bundler.clean_system(command)
+ end
+ end
- version_string = "v#{Rubydoop::VERSION}"
-
- unless %x(git tag -l).split("\n").include?(version_string)
- system %(git tag -a #{version_string} -m #{version_string})
+ task :classpath do
+ File.open('spec/hadoop_setup.rb', 'w') do |io|
+ hadoop_home = File.expand_path(Dir["tmp/hadoop*"].first)
+ %x(#{hadoop_home}/bin/hadoop classpath).chomp.split(':').each do |pattern|
+ Dir[pattern].each do |path|
+ io.puts("$CLASSPATH << '#{File.expand_path(path)}'")
+ end
+ end
+ io.puts("HADOOP_HOME = '#{hadoop_home}'")
+ end
end
+end
+
+desc 'Download Hadoop and set up classpath'
+task :setup => ['setup:hadoop', 'setup:test_project', 'setup:classpath']
+
+require 'rspec/core/rake_task'
- system %(git push && git push --tags; gem build rubydoop.gemspec && gem push rubydoop-*.gem && mv rubydoop-*.gem pkg)
+RSpec::Core::RakeTask.new(:spec) do |r|
+ r.rspec_opts = '--tty'
end
+
+task :spec => :dist
+
+require 'bundler'
+
+Bundler::GemHelper.install_tasks
@@ -14,14 +14,9 @@ module JavaLang
end
describe 'Packaging and running a project' do
- def gemset(dir)
- rvmrc = File.readlines("#{dir}/.rvmrc").first
- rvmrc.scan(/(\S+@\S+)/).flatten.first
- end
-
def isolated_run(dir, cmd)
Dir.chdir(dir) do
- system("BUNDLE_GEMFILE='' rvm #{gemset(dir)} exec #{cmd}")
+ Bundler.clean_system("rvm $RUBY_VERSION@rubydoop-test_project do #{cmd}")
end
end
@@ -30,7 +25,7 @@ def isolated_run(dir, cmd)
end
before :all do
- isolated_run(test_project_dir, './bin/rake clean package')
+ isolated_run(test_project_dir, 'rake clean package')
end
around do |example|
@@ -73,7 +68,7 @@ def isolated_run(dir, cmd)
end
it 'includes jruby-complete.jar' do
- jruby_version = gemset(test_project_dir).scan(/jruby-(.+)@/).flatten.first
+ jruby_version = ENV['RUBY_VERSION'].scan(/^jruby-(.+)$/).flatten.first
jar_entries.should include("lib/jruby-complete-#{jruby_version}.jar")
end
@@ -96,7 +91,7 @@ def isolated_run(dir, cmd)
context 'Running the project' do
before :all do
- isolated_run(test_project_dir, "hadoop jar build/test_project.jar -conf conf/hadoop-local.xml test_project data/input data/output 2>&1 | tee data/log")
+ isolated_run(test_project_dir, "#{HADOOP_HOME}/bin/hadoop jar build/test_project.jar -conf conf/hadoop-local.xml test_project data/input data/output 2>&1 | tee data/log")
end
let :log do
@@ -1,2 +0,0 @@
----
-BUNDLE_BIN: bin
@@ -1 +0,0 @@
-rvm --create use jruby-1.6.7@rubydoop_test_project
@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org/'
gem 'rubydoop', :path => '../../..'
gem 'json'
@@ -4,7 +4,7 @@ PATH
rubydoop (1.1.0)
GEM
- remote: http://rubygems.org/
+ remote: https://rubygems.org/
specs:
bouncy-castle-java (1.5.0146.1)
coderay (1.0.8)
View
@@ -1,10 +1,8 @@
# encoding: utf-8
-require 'java'
-
-IO.foreach(File.expand_path('../../.classpath', __FILE__)) { |path| $CLASSPATH << path.chomp }
-
require 'pry'
+require 'java'
+require 'hadoop_setup'
require 'rubydoop'
require 'rubydoop/version'
require 'rubydoop/package'

0 comments on commit e921273

Please sign in to comment.