Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
Checking mergeability… Don't worry, you can still create the pull request.
  • 20 commits
  • 20 files changed
  • 0 commit comments
  • 3 contributors
Commits on Apr 09, 2012
romand-ironio Merge pull request #25 from romand-ironio/more-examples
add way to run separate example
cde0130
Commits on Apr 10, 2012
romand-ironio Merge pull request #26 from romand-ironio/more-examples
adding simple example to cover more of ng
8869fc5
romand-ironio Merge pull request #27 from romand-ironio/more-examples
adding first basic test
620111d
@iced iced Worker presence checks in base code and using __runner__ instead of r…
…unner for runners.
70200b8
@iced iced Merge branch 'master' of github.com:iron-io/iron_worker_ruby_ng 8cb7fd2
@iced iced Removing test related things from main Rakefile/Gemfile c4cbe94
@iced iced Using runner func instead of hardcoded runner name when creating runn…
…ers.
d7e823c
@iced iced Debug output removed. b15c692
@iced iced Switching node to sh runtime. a3dd1cd
@iced iced Switching java to sh runtime. a984cb5
@iced iced Binary worker implemented. a58e639
Commits on Apr 12, 2012
@iced iced Experimental support for merging binary gems on demand. c990265
Commits on Apr 13, 2012
@romand providing binary libraries from worker dir 20635f9
romand-ironio Merge pull request #28 from romand-ironio/master
providing binary libraries from worker dir
baffe1c
Commits on Apr 15, 2012
@iced iced Typo fixed and version bump. aa88673
@iced iced Regenerate gemspec for version 0.2.1 d9b2d53
@iced iced Removing obsolet log. d7de45c
@iced iced Removing obsolet log - examples. e0f5b9c
@iced iced Regenerate gemspec for version 0.2.2 1e9a02b
Commits on Apr 16, 2012
@romand fixing merge_file feature with test bd0b79e
View
6 Gemfile
@@ -9,9 +9,3 @@ gem 'bundler', '> 1.0.0'
group :development do
gem 'jeweler', '~> 1.8.3'
end
-
-group :test do
- gem 'iron_worker_ng'
- gem 'typhoeus'
- gem 'pry'
-end
View
20 Gemfile.lock
@@ -1,26 +1,14 @@
GEM
remote: http://rubygems.org/
specs:
- coderay (1.0.6)
git (1.2.5)
- iron_worker_ng (0.2.0)
- bundler (> 1.0.0)
- json
- rest
- rest-client
- zip
jeweler (1.8.3)
bundler (~> 1.0)
git (>= 1.2.5)
rake
rdoc
- json (1.6.5)
- method_source (0.7.1)
+ json (1.6.6)
mime-types (1.18)
- pry (0.9.8.4)
- coderay (~> 1.0.5)
- method_source (~> 0.7.1)
- slop (>= 2.4.4, < 3)
rake (0.9.2.2)
rdoc (3.12)
json (~> 1.4)
@@ -29,9 +17,6 @@ GEM
rest-client
rest-client (1.6.7)
mime-types (>= 1.16)
- slop (2.4.4)
- typhoeus (0.3.3)
- mime-types
zip (2.0.2)
PLATFORMS
@@ -39,11 +24,8 @@ PLATFORMS
DEPENDENCIES
bundler (> 1.0.0)
- iron_worker_ng
jeweler (~> 1.8.3)
json
- pry
rest
rest-client
- typhoeus
zip
View
45 Rakefile
@@ -1,49 +1,4 @@
require 'rubygems'
-
-require 'rake/testtask'
-require 'tmpdir'
-
-Rake::TestTask.new do |t|
- examples_tests_dir = Dir.mktmpdir('iw_examples')
-
- FileUtils::cp_r(Dir.glob('examples/*'), examples_tests_dir)
-
- Dir.glob('examples/**/**.rb').each do |path|
- next unless path =~ %r|examples/(.*)/([^/]+)/\2.rb$|
-
- dir = $1
- name = $2
-
- test_path = examples_tests_dir + "/#{dir}/#{name}/test_#{name}.rb"
-
- File.open(test_path, 'w') do |out|
- out << "require 'helpers'\n"
- out << "class #{name.capitalize}Test < Test::Unit::TestCase\n"
- out << "def test_example\n"
-
- File.readlines(path).each do |line|
- line, assert_str = line.chomp.split /#>/
- out << line << "\n"
-
- if assert_str
- cond, desc = assert_str.split /--/
- out << "assert(" << cond << ", '" <<
- (desc or "").gsub(/'/, "\\\\'") << "')\n"
- end
- end
-
- out << "end\nend\n"
- end
- end
-
- t.libs << "lib" << "test" << examples_tests_dir
- files = FileList['test/**/**.rb',
- examples_tests_dir + '/**/test_*.rb']
- t.test_files = files.keep_if{ |f| f =~ Regexp.new(ENV['TESTP'] || '') }
-
- t.verbose = true
-end
-
require 'bundler'
require 'jeweler'
View
2  VERSION
@@ -1 +1 @@
-0.2.0
+0.2.2
View
10 examples/ruby/master_slave/master_worker.rb
@@ -4,19 +4,19 @@
client = IronWorkerNG::Client.new(:token => iron_io_token,
:project_id => iron_io_project_id)
-log 'Running slave workers...'
+puts 'Running slave workers...'
task_ids = []
params[:args].each do |arg|
- log "Queueing slave with arg=#{arg.to_s}"
+ puts "Queueing slave with arg=#{arg.to_s}"
task_ids << client.tasks.create('SlaveWorker', { :arg => arg }).id
end
-log 'Retriving results from slaves logs'
+puts 'Retriving results from slaves logs'
results = task_ids.map do |id|
client.tasks.wait_for(id)
client.tasks.log(id).to_i
end
-log "Sum = #{ results.inject(:+) }"
+puts "Sum = #{ results.inject(:+) }"
-log 'Done'
+puts 'Done'
View
2  examples/ruby/master_slave/slave_worker.rb
@@ -1 +1 @@
-log params[:arg].inject(:+)
+puts params[:arg].inject(:+)
View
13 iron_worker_ng.gemspec
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = "iron_worker_ng"
- s.version = "0.2.0"
+ s.version = "0.2.2"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Andrew Kirilenko", "Iron.io, Inc"]
- s.date = "2012-04-09"
+ s.date = "2012-04-15"
s.description = "New generation ruby client for IronWorker"
s.email = "info@iron.io"
s.executables = ["iron_worker_ng"]
@@ -27,15 +27,20 @@ Gem::Specification.new do |s|
"examples/ruby/master_slave/master_slave.rb",
"examples/ruby/master_slave/master_worker.rb",
"examples/ruby/master_slave/slave_worker.rb",
+ "examples/ruby/simple/Gemfile",
+ "examples/ruby/simple/sample_worker.rb",
+ "examples/ruby/simple/simple.rb",
"lib/iron_worker_ng.rb",
"lib/iron_worker_ng/api_client.rb",
"lib/iron_worker_ng/api_client_error.rb",
"lib/iron_worker_ng/client.rb",
"lib/iron_worker_ng/code/base.rb",
+ "lib/iron_worker_ng/code/binary.rb",
"lib/iron_worker_ng/code/java.rb",
"lib/iron_worker_ng/code/node.rb",
"lib/iron_worker_ng/code/ruby.rb",
"lib/iron_worker_ng/feature/base.rb",
+ "lib/iron_worker_ng/feature/binary/merge_worker.rb",
"lib/iron_worker_ng/feature/common/merge_dir.rb",
"lib/iron_worker_ng/feature/common/merge_file.rb",
"lib/iron_worker_ng/feature/java/merge_jar.rb",
@@ -46,7 +51,9 @@ Gem::Specification.new do |s|
"lib/iron_worker_ng/feature/ruby/merge_worker.rb",
"lib/iron_worker_ng/logger.rb",
"lib/iron_worker_ng/version.rb",
- "test/helpers.rb"
+ "test/hello.rb",
+ "test/helpers.rb",
+ "test/test_basic.rb"
]
s.homepage = "https://github.com/iron-io/iron_worker_ruby_ng"
s.require_paths = ["lib"]
View
1  lib/iron_worker_ng.rb
@@ -5,3 +5,4 @@
require_relative 'iron_worker_ng/code/ruby'
require_relative 'iron_worker_ng/code/java'
require_relative 'iron_worker_ng/code/node'
+require_relative 'iron_worker_ng/code/binary'
View
15 lib/iron_worker_ng/code/base.rb
@@ -60,23 +60,20 @@ def bundle(zip)
end
def create_zip
- fixate
-
- init_code = ''
-
- @features.each do |f|
- if f.respond_to?(:code_for_init)
- init_code += f.send(:code_for_init) + "\n"
- end
+ unless @worker
+ IronWorkerNG::Logger.error 'No worker specified'
+ raise 'No worker specified'
end
+ fixate
+
zip_name = Dir.tmpdir + '/' + Dir::Tmpname.make_tmpname("iron-worker-ng-", "code.zip")
IronWorkerNG::Logger.debug "Creating code zip '#{zip_name}'"
Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zip|
bundle(zip)
- create_runner(zip, init_code)
+ create_runner(zip)
end
zip_name
View
43 lib/iron_worker_ng/code/binary.rb
@@ -0,0 +1,43 @@
+require_relative '../feature/binary/merge_worker'
+
+module IronWorkerNG
+ module Code
+ class Binary < IronWorkerNG::Code::Base
+ include IronWorkerNG::Feature::Binary::MergeWorker::InstanceMethods
+
+ def create_runner(zip)
+ zip.get_output_stream(runner) do |runner|
+ runner.write <<RUNNER
+#!/bin/sh
+# iron_worker_ng-#{IronWorkerNG.version}
+
+root() {
+ while [ $# -gt 0 ]; do
+ if [ "$1" = "-d" ]; then
+ printf "%s\n" "$2"
+ break
+ fi
+ done
+}
+
+cd "$(root "$@")"
+
+chmod +x #{File.basename(worker.path)}
+
+LD_LIBRARY_PATH=. ./#{File.basename(worker.path)} "$@"
+RUNNER
+ end
+ end
+
+ def runtime
+ 'sh'
+ end
+
+ def runner
+ '__runner__.sh'
+ end
+ end
+ end
+end
+
+IronWorkerNG::Code::Base.register_type(:name => 'binary', :klass => IronWorkerNG::Code::Binary)
View
35 lib/iron_worker_ng/code/java.rb
@@ -7,14 +7,7 @@ class Java < IronWorkerNG::Code::Base
include IronWorkerNG::Feature::Java::MergeJar::InstanceMethods
include IronWorkerNG::Feature::Java::MergeWorker::InstanceMethods
- def create_runner(zip, init_code)
- IronWorkerNG::Logger.info 'Creating java runner'
-
- unless @worker
- IronWorkerNG::Logger.error 'No worker specified'
- raise 'No worker specified'
- end
-
+ def create_runner(zip)
classpath_array = []
@features.each do |f|
@@ -27,31 +20,33 @@ def create_runner(zip, init_code)
IronWorkerNG::Logger.info "Collected '#{classpath}' classpath"
- zip.get_output_stream('runner.rb') do |runner|
+ zip.get_output_stream(runner) do |runner|
runner.write <<RUNNER
+#!/bin/sh
# iron_worker_ng-#{IronWorkerNG.version}
-root = nil
-
-($*.length - 2).downto(0) do |i|
- root = $*[i + 1] if $*[i] == '-d'
-end
-
-Dir.chdir(root)
+root() {
+ while [ $# -gt 0 ]; do
+ if [ "$1" = "-d" ]; then
+ printf "%s\n" "$2"
+ break
+ fi
+ done
+}
-#{init_code}
+cd "$(root "$@")"
-puts `java -cp #{classpath} #{worker.klass} \#{$*.join(' ')}`
+java -cp #{classpath} #{worker.klass} "$@"
RUNNER
end
end
def runtime
- 'ruby'
+ 'sh'
end
def runner
- 'runner.rb'
+ '__runner__.sh'
end
end
end
View
35 lib/iron_worker_ng/code/node.rb
@@ -5,39 +5,34 @@ module Code
class Node < IronWorkerNG::Code::Base
include IronWorkerNG::Feature::Node::MergeWorker::InstanceMethods
- def create_runner(zip, init_code)
- IronWorkerNG::Logger.info 'Creating node runner'
-
- unless @worker
- IronWorkerNG::Logger.error 'No worker specified'
- raise 'No worker specified'
- end
-
- zip.get_output_stream('runner.rb') do |runner|
+ def create_runner(zip)
+ zip.get_output_stream(runner) do |runner|
runner.write <<RUNNER
+#!/bin/sh
# iron_worker_ng-#{IronWorkerNG.version}
-root = nil
-
-($*.length - 2).downto(0) do |i|
- root = $*[i + 1] if $*[i] == '-d'
-end
-
-Dir.chdir(root)
+root() {
+ while [ $# -gt 0 ]; do
+ if [ "$1" = "-d" ]; then
+ printf "%s\n" "$2"
+ break
+ fi
+ done
+}
-#{init_code}
+cd "$(root "$@")"
-puts `node \#{worker_file_name} \#{$*.join(' ')}`
+node #{File.basename(worker.path)} "$@"
RUNNER
end
end
def runtime
- 'ruby'
+ 'sh'
end
def runner
- 'runner.rb'
+ '__runner__.sh'
end
end
end
View
31 lib/iron_worker_ng/code/ruby.rb
@@ -9,15 +9,18 @@ class Ruby < IronWorkerNG::Code::Base
include IronWorkerNG::Feature::Ruby::MergeGemfile::InstanceMethods
include IronWorkerNG::Feature::Ruby::MergeWorker::InstanceMethods
- def create_runner(zip, init_code)
- IronWorkerNG::Logger.info 'Creating ruby runner'
-
- unless @worker
- IronWorkerNG::Logger.error 'No worker specified'
- raise 'No worker specified'
+ def create_runner(zip)
+ gempath_code_array = []
+
+ @features.each do |f|
+ if f.respond_to?(:code_for_gempath)
+ gempath_code_array << f.send(:code_for_gempath)
+ end
end
- zip.get_output_stream('runner.rb') do |runner|
+ gempath_code = gempath_code_array.join("\n")
+
+ zip.get_output_stream(runner) do |runner|
runner.write <<RUNNER
# iron_worker_ng-#{IronWorkerNG.version}
@@ -25,7 +28,7 @@ def create_runner(zip, init_code)
payload_file = nil
task_id = nil
-($*.length - 2).downto(0) do |i|
+0.upto($*.length - 2) do |i|
root = $*[i + 1] if $*[i] == '-d'
payload_file = $*[i + 1] if $*[i] == '-payload'
task_id = $*[i + 1] if $*[i] == '-id'
@@ -33,13 +36,9 @@ def create_runner(zip, init_code)
Dir.chdir(root)
-#{init_code}
+#{gempath_code}
$:.unshift("\#{root}")
-def log(*args)
- puts *args
-end
-
require 'json'
@iron_worker_task_id = task_id
@@ -81,12 +80,12 @@ def params
@params
end
-require worker_file_name
+require '#{File.basename(worker.path)}'
worker_class = nil
begin
- worker_class = Kernel.const_get(worker_class_name)
+ worker_class = Kernel.const_get('#{worker.klass}')
rescue
end
@@ -112,7 +111,7 @@ def runtime
end
def runner
- 'runner.rb'
+ '__runner__.rb'
end
end
end
View
50 lib/iron_worker_ng/feature/binary/merge_worker.rb
@@ -0,0 +1,50 @@
+module IronWorkerNG
+ module Feature
+ module Binary
+ module MergeWorker
+ class Feature < IronWorkerNG::Feature::Base
+ attr_reader :path
+
+ def initialize(path)
+ @path = File.expand_path(path)
+ end
+
+ def hash_string
+ Digest::MD5.hexdigest(@path + File.mtime(@path).to_i.to_s)
+ end
+
+ def bundle(zip)
+ IronWorkerNG::Logger.debug "Bundling binary worker with path='#{@path}'"
+
+ zip.add(File.basename(@path), @path)
+ end
+ end
+
+ module InstanceMethods
+ attr_reader :worker
+
+ def merge_worker(path)
+ @worker ||= nil
+
+ unless @worker.nil?
+ IronWorkerNG::Logger.warn "Ignoring attempt to merge binary worker with path='#{path}'"
+ return
+ end
+
+ @name ||= File.basename(path).gsub(/\..*$/, '').capitalize.gsub(/_./) { |x| x[1].upcase }
+
+ @worker = IronWorkerNG::Feature::Binary::MergeWorker::Feature.new(path)
+
+ IronWorkerNG::Logger.info "Merging binary worker with path='#{path}'"
+
+ @features << @worker
+ end
+
+ def self.included(base)
+ IronWorkerNG::Code::Base.register_feature(:name => 'merge_worker', :for_klass => base, :args => 'PATH')
+ end
+ end
+ end
+ end
+ end
+end
View
2  lib/iron_worker_ng/feature/common/merge_file.rb
@@ -18,7 +18,7 @@ def hash_string
def bundle(zip)
IronWorkerNG::Logger.debug "Bundling file with path='#{@path}' and dest='#{@dest}'"
- zip.add(@dest + File.basename(@path), @path)
+ zip.add(@dest + '/' + File.basename(@path), @path)
end
end
View
4 lib/iron_worker_ng/feature/node/merge_worker.rb
@@ -18,10 +18,6 @@ def bundle(zip)
zip.add(File.basename(@path), @path)
end
-
- def code_for_init
- "worker_file_name = '#{File.basename(@path)}'"
- end
end
module InstanceMethods
View
12 lib/iron_worker_ng/feature/ruby/merge_gem.rb
@@ -4,6 +4,14 @@ module IronWorkerNG
module Feature
module Ruby
module MergeGem
+ def self.merge_binary?
+ @merge_binary ||= false
+ end
+
+ def self.merge_binary=(merge_binary)
+ @merge_binary = merge_binary
+ end
+
class Feature < IronWorkerNG::Feature::Base
attr_reader :spec
@@ -31,7 +39,7 @@ def hash_string
end
def bundle(zip)
- if @spec.extensions.length == 0
+ if @spec.extensions.length == 0 || IronWorkerNG::Feature::Ruby::MergeGem.merge_binary?
IronWorkerNG::Logger.debug "Bundling ruby gem with name='#{@spec.name}' and version='#{@spec.version}'"
zip.add('gems/' + @spec.full_name, gem_path)
@@ -43,7 +51,7 @@ def bundle(zip)
end
end
- def code_for_init
+ def code_for_gempath
if @spec.extensions.length == 0
'$:.unshift("#{root}/gems/' + @spec.full_name + '/lib")'
else
View
4 lib/iron_worker_ng/feature/ruby/merge_worker.rb
@@ -20,10 +20,6 @@ def bundle(zip)
zip.add(File.basename(@path), @path)
end
-
- def code_for_init
- "worker_file_name = '#{File.basename(@path)}'\nworker_class_name='#{@klass}'"
- end
end
module InstanceMethods
View
37 test/helpers.rb
@@ -1 +1,38 @@
require 'test/unit'
+require 'tempfile'
+
+require './lib/iron_worker_ng.rb'
+
+def code_bundle(name,&block)
+ code = IronWorkerNG::Code::Ruby.new(name)
+
+ class << code
+ def worker_code(str)
+ Tempfile.open('worker') do |f|
+ f << str
+ f.close
+ merge_worker(f.path)
+ f.unlink
+ end
+ end
+ end
+
+ code.instance_eval(&block)
+
+ code
+end
+
+class IWNGTest < Test::Unit::TestCase
+ attr_accessor :client
+
+ def setup
+ IronWorkerNG::Logger.logger.level = ::Logger::DEBUG
+
+ token, project_id = [ ENV['IRON_IO_TOKEN'], ENV['IRON_IO_PROJECT_ID'] ]
+ raise("please set $IRON_IO_TOKEN and $IRON_IO_PROJECT_ID " +
+ "environment variables") unless token and project_id
+
+ @client = IronWorkerNG::Client.new(:token => token,
+ :project_id => project_id )
+ end
+end
View
42 test/test_basic.rb
@@ -1,21 +1,7 @@
-require './lib/iron_worker_ng.rb'
-require 'test/unit'
+require 'helpers'
-class BasicTest < Test::Unit::TestCase
- attr_accessor :client
-
- def setup
- IronWorkerNG::Logger.logger.level = ::Logger::DEBUG
-
- token, project_id = [ ENV['IRON_IO_TOKEN'], ENV['IRON_IO_PROJECT_ID'] ]
- raise("please set $IRON_IO_TOKEN and $IRON_IO_PROJECT_ID " +
- "environment variables") unless token and project_id
-
- @client = IronWorkerNG::Client.new(:token => token,
- :project_id => project_id )
- end
-
- def test_basic
+class BasicTest < IWNGTest
+ def _test_basic
code = IronWorkerNG::Code::Ruby.new('test_basic')
code.merge_worker(File.dirname(__FILE__) + '/hello.rb')
client.codes_create(code)
@@ -25,13 +11,23 @@ def test_basic
assert_equal( "hello\n", log, "worker stdout is in log" )
end
- def test_30_codes
- 31.times do |i|
- code = IronWorkerNG::Code::Ruby.new("test_30_codes_code#{i}")
- code.merge_worker(File.dirname(__FILE__) + '/hello.rb')
- client.codes_create(code)
+ def test_symlinks
+ Dir.unlink './test/data/dir1/dir2' if
+ Dir.exist? './test/data/dir1/dir2'
+ Dir.chdir('test/data/dir1') do
+ File.symlink('./test/data/dir2', 'dir2')
end
- assert_equal( 31, client.codes_list.size )
+
+ code = code_bundle 'test_symlinks' do
+ merge_dir('data/dir1', 'data')
+ worker 'puts File.read("dir1/dir2/test")'
+ end
+
+ puts code.create_zip
+
+ File.unlink 'test/data/dir1/dir2'
+
+ assert true
end
end

No commit comments for this range

Something went wrong with that request. Please try again.