From 59184180bdc359326a5b559a0331bd4ff436cb92 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 17:08:20 +0200 Subject: [PATCH 1/8] add manticore transport --- .../transport/transport/http/manticore.rb | 64 +++++++++++++ .../test/unit/transport_manticore_test.rb | 96 +++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb create mode 100644 elasticsearch-transport/test/unit/transport_manticore_test.rb diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb new file mode 100644 index 0000000000..a8d5155a5e --- /dev/null +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -0,0 +1,64 @@ +require 'manticore' + +module Elasticsearch + module Transport + module Transport + module HTTP + class Manticore + include Base + def perform_request(method, path, params={}, body=nil) + super do |connection, url| + params[:body] = __convert_to_json(body) if body + case method + when "GET" + resp = connection.connection.get(url, params) + when "HEAD" + resp = connection.connection.head(url, params) + when "PUT" + resp = connection.connection.put(url, params) + when "POST" + resp = connection.connection.post(url, params) + when "DELETE" + resp = connection.connection.delete(url, params) + else + raise ArgumentError.new "Method #{method} not supported" + end + Response.new resp.code, resp.read_body, resp.headers + end + end + def __build_connections + # TODO: not threadsafe + ssl_options = options[:transport_options][:ssl] || {} + if ssl_options[:truststore] + java.lang.System.setProperty "javax.net.ssl.trustStore", ssl_options[:truststore] + end + if ssl_options[:truststore_password] + java.lang.System.setProperty "javax.net.ssl.trustStorePassword", ssl_options[:truststore_password] + end + if ssl_options[:verify] == false + options[:transport_options][:ignore_ssl_validation] = true + end + Connections::Collection.new \ + :connections => hosts.map { |host| + host[:protocol] = host[:scheme] || DEFAULT_PROTOCOL + host[:port] ||= DEFAULT_PORT + url = __full_url(host) + + Connections::Connection.new \ + :host => host, + :connection => ::Manticore::Client.new(options[:transport_options] || {}) + }, + :selector_class => options[:selector_class], + :selector => options[:selector] + end + + def host_unreachable_exceptions + [ + # TODO: list HTTPClient exceptions + ] + end + end + end + end + end +end diff --git a/elasticsearch-transport/test/unit/transport_manticore_test.rb b/elasticsearch-transport/test/unit/transport_manticore_test.rb new file mode 100644 index 0000000000..bb21e98747 --- /dev/null +++ b/elasticsearch-transport/test/unit/transport_manticore_test.rb @@ -0,0 +1,96 @@ +require_relative '../test_helper' + +if !JRUBY + puts "'#{File.basename(__FILE__)}' only supported on JRuby (you're running #{RUBY_VERSION})" +else + require 'elasticsearch/transport/transport/http/manticore' + require 'manticore' + + class Elasticsearch::Transport::Transport::HTTP::ManticoreTest < Test::Unit::TestCase + include Elasticsearch::Transport::Transport::HTTP + + context "Manticore transport" do + setup do + @transport = Manticore.new :hosts => [ { :host => '127.0.0.1', :port => 8080 } ] + end + + should "implement host_unreachable_exceptions" do + assert_instance_of Array, @transport.host_unreachable_exceptions + end + + should "implement __build_connections" do + assert_equal 1, @transport.hosts.size + assert_equal 1, @transport.connections.size + + assert_instance_of ::Manticore::Client, @transport.connections.first.connection + end + + should "perform the request" do + @transport.connections.first.connection.expects(:get).returns(stub_everything) + @transport.perform_request 'GET', '/' + end + + should "set body for GET request" do + @transport.connections.first.connection.expects(:get). + with('http://127.0.0.1:8080//', {:body => '{"foo":"bar"}'}).returns(stub_everything) + @transport.perform_request 'GET', '/', {}, '{"foo":"bar"}' + end + + should "set body for PUT request" do + @transport.connections.first.connection.expects(:put). + with('http://127.0.0.1:8080//', {:body => '{"foo":"bar"}'}).returns(stub_everything) + @transport.perform_request 'PUT', '/', {}, {:foo => 'bar'} + end + + should "serialize the request body" do + @transport.connections.first.connection.expects(:post). + with('http://127.0.0.1:8080//', {:body => '{"foo":"bar"}'}).returns(stub_everything) + @transport.perform_request 'POST', '/', {}, {'foo' => 'bar'} + end + + should "not serialize a String request body" do + @transport.connections.first.connection.expects(:post). + with('http://127.0.0.1:8080//', {:body => '{"foo":"bar"}'}).returns(stub_everything) + @transport.serializer.expects(:dump).never + @transport.perform_request 'POST', '/', {}, '{"foo":"bar"}' + end + + should "set application/json header" do + @transport.connections.first.connection.stub("http://127.0.0.1:8080", body: '{"foo":"bar"}', headers: {"X-Content-Type" => "application/json"}, code: 200 ) + #@transport.connections.first.connection.expects(:get).returns(stub_everything) + + response = @transport.perform_request 'GET', '/', { :headers => {"X-Content-Type" => "application/json"} } + + assert_equal 'application/json', response.headers['content-type'] + end + + should "handle HTTP methods" do + @transport.connections.first.connection.expects(:delete).with('http://127.0.0.1:8080//', {}).returns(stub_everything) + @transport.connections.first.connection.expects(:head).with('http://127.0.0.1:8080//', {}).returns(stub_everything) + @transport.connections.first.connection.expects(:get).with('http://127.0.0.1:8080//', {}).returns(stub_everything) + @transport.connections.first.connection.expects(:put).with('http://127.0.0.1:8080//', {}).returns(stub_everything) + @transport.connections.first.connection.expects(:post).with('http://127.0.0.1:8080//', {}).returns(stub_everything) + + %w| HEAD GET PUT POST DELETE |.each { |method| @transport.perform_request method, '/' } + + assert_raise(ArgumentError) { @transport.perform_request 'FOOBAR', '/' } + end + + should "allow to set options for Manticore" do + transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234 } ] do |curl| + curl.headers["User-Agent"] = "myapp-0.0" + end + + assert_equal "myapp-0.0", transport.connections.first.connection.headers["User-Agent"] + end + + should "set the credentials if passed" do + transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234, :user => 'foo', :password => 'bar' } ] + assert_equal 'foo', transport.connections.first.connection.username + assert_equal 'bar', transport.connections.first.connection.password + end + end + + end + +end From f1859f27151b26b30967c73991aebc268e7d0bef Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 17:30:32 +0200 Subject: [PATCH 2/8] add list of exceptions for manticore transport --- .../lib/elasticsearch/transport/transport/http/manticore.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb index a8d5155a5e..3701b14b5c 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -54,7 +54,10 @@ def __build_connections def host_unreachable_exceptions [ - # TODO: list HTTPClient exceptions + Manticore::Timeout, + Manticore::SocketException, + Manticore::ClientProtocolException, + Manticore::ResolutionFailure ] end end From 4816edc3d187ccd522a77c844781eab07396e4ef Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 18:18:12 +0200 Subject: [PATCH 3/8] correct names of manticore exceptions --- .../elasticsearch/transport/transport/http/manticore.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb index 3701b14b5c..01477cb694 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -54,10 +54,10 @@ def __build_connections def host_unreachable_exceptions [ - Manticore::Timeout, - Manticore::SocketException, - Manticore::ClientProtocolException, - Manticore::ResolutionFailure + ::Manticore::Timeout, + ::Manticore::SocketException, + ::Manticore::ClientProtocolException, + ::Manticore::ResolutionFailure ] end end From 544366f7acb3d97a68220e36eee1aeb5396742dd Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 18:25:30 +0200 Subject: [PATCH 4/8] add basic preambles to manticore transport --- .../transport/transport/http/manticore.rb | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb index 01477cb694..df2442d3af 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -4,8 +4,20 @@ module Elasticsearch module Transport module Transport module HTTP + # Alternative HTTP transport implementation, using the [_Manticore_](https://github.com/cheald/manticore) client. + # + # This transport works only on JRuby + # + # @see Transport::Base + # class Manticore include Base + + # Performs the request by invoking {Transport::Base#perform_request} with a block. + # + # @return [Response] + # @see Transport::Base#perform_request + # def perform_request(method, path, params={}, body=nil) super do |connection, url| params[:body] = __convert_to_json(body) if body @@ -26,6 +38,12 @@ def perform_request(method, path, params={}, body=nil) Response.new resp.code, resp.read_body, resp.headers end end + + # Builds and returns a collection of connections. + # Each connection is a Manticore::Client + # + # @return [Connections::Collection] + # def __build_connections # TODO: not threadsafe ssl_options = options[:transport_options][:ssl] || {} @@ -52,6 +70,10 @@ def __build_connections :selector => options[:selector] end + # Returns an array of implementation specific connection errors. + # + # @return [Array] + # def host_unreachable_exceptions [ ::Manticore::Timeout, From 7905e2622ffa22d0ba65e4ef8f0bf0df60d3f345 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 18:27:03 +0200 Subject: [PATCH 5/8] small bugfix on manticore __build_connections --- .../lib/elasticsearch/transport/transport/http/manticore.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb index df2442d3af..6c91fc20b6 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -46,7 +46,8 @@ def perform_request(method, path, params={}, body=nil) # def __build_connections # TODO: not threadsafe - ssl_options = options[:transport_options][:ssl] || {} + transport_options = options[:transport_options] || {} + ssl_options = transport_options[:ssl] || {} if ssl_options[:truststore] java.lang.System.setProperty "javax.net.ssl.trustStore", ssl_options[:truststore] end From d36786d73929673b498d3d92a7acf12e6e5c68e1 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 18:27:29 +0200 Subject: [PATCH 6/8] fixed typo on manticore test --- elasticsearch-transport/test/unit/transport_manticore_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch-transport/test/unit/transport_manticore_test.rb b/elasticsearch-transport/test/unit/transport_manticore_test.rb index bb21e98747..1530eb6a35 100644 --- a/elasticsearch-transport/test/unit/transport_manticore_test.rb +++ b/elasticsearch-transport/test/unit/transport_manticore_test.rb @@ -77,8 +77,8 @@ class Elasticsearch::Transport::Transport::HTTP::ManticoreTest < Test::Unit::Tes end should "allow to set options for Manticore" do - transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234 } ] do |curl| - curl.headers["User-Agent"] = "myapp-0.0" + transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234 } ] do |manticore| + manticore.headers["User-Agent"] = "myapp-0.0" end assert_equal "myapp-0.0", transport.connections.first.connection.headers["User-Agent"] From b4432e1a28274c9f263dc26bf9d8fe2b447eac40 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 6 Oct 2014 19:03:46 +0200 Subject: [PATCH 7/8] add manticore dev dependency --- elasticsearch-transport/elasticsearch-transport.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch-transport/elasticsearch-transport.gemspec b/elasticsearch-transport/elasticsearch-transport.gemspec index b0a874402f..9268324697 100644 --- a/elasticsearch-transport/elasticsearch-transport.gemspec +++ b/elasticsearch-transport/elasticsearch-transport.gemspec @@ -46,6 +46,7 @@ Gem::Specification.new do |s| s.add_development_dependency "curb" unless defined? JRUBY_VERSION s.add_development_dependency "patron" unless defined? JRUBY_VERSION s.add_development_dependency "typhoeus", '~> 0.6' + s.add_development_dependency "manticore" if defined? JRUBY_VERSION # Prevent unit test failures on Ruby 1.8 if defined?(RUBY_VERSION) && RUBY_VERSION < '1.9' From 9c0cbb829170fd324c2e2a44e29c2131da1624f7 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 9 Oct 2014 10:57:34 +0200 Subject: [PATCH 8/8] added ssl tests and header configuration for manticore --- .../transport/transport/http/manticore.rb | 41 +++++++++++++------ .../test/unit/transport_manticore_test.rb | 38 +++++++++++------ 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb index 6c91fc20b6..b482304889 100644 --- a/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb +++ b/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb @@ -21,6 +21,7 @@ class Manticore def perform_request(method, path, params={}, body=nil) super do |connection, url| params[:body] = __convert_to_json(body) if body + params = params.merge @request_options case method when "GET" resp = connection.connection.get(url, params) @@ -45,27 +46,27 @@ def perform_request(method, path, params={}, body=nil) # @return [Connections::Collection] # def __build_connections - # TODO: not threadsafe - transport_options = options[:transport_options] || {} - ssl_options = transport_options[:ssl] || {} - if ssl_options[:truststore] - java.lang.System.setProperty "javax.net.ssl.trustStore", ssl_options[:truststore] - end - if ssl_options[:truststore_password] - java.lang.System.setProperty "javax.net.ssl.trustStorePassword", ssl_options[:truststore_password] - end - if ssl_options[:verify] == false - options[:transport_options][:ignore_ssl_validation] = true + @request_options = {} + + if options.key?(:headers) + @request_options[:headers] = options[:headers] end + + client_options = setup_ssl(options[:ssl] || {}) + Connections::Collection.new \ :connections => hosts.map { |host| host[:protocol] = host[:scheme] || DEFAULT_PROTOCOL host[:port] ||= DEFAULT_PORT + + host.delete(:user) # auth is not supported here. + host.delete(:password) # use the headers + url = __full_url(host) Connections::Connection.new \ :host => host, - :connection => ::Manticore::Client.new(options[:transport_options] || {}) + :connection => ::Manticore::Client.new(:options => client_options) }, :selector_class => options[:selector_class], :selector => options[:selector] @@ -83,6 +84,22 @@ def host_unreachable_exceptions ::Manticore::ResolutionFailure ] end + + private + # TODO: not threadsafe + def setup_ssl(ssl_options) + if ssl_options[:truststore] + java.lang.System.setProperty "javax.net.ssl.trustStore", ssl_options[:truststore] + end + if ssl_options[:truststore_password] + java.lang.System.setProperty "javax.net.ssl.trustStorePassword", ssl_options[:truststore_password] + end + if ssl_options[:verify] == false then + { :ignore_ssl_validation => true } + else + {} + end + end end end end diff --git a/elasticsearch-transport/test/unit/transport_manticore_test.rb b/elasticsearch-transport/test/unit/transport_manticore_test.rb index 1530eb6a35..6673e986d8 100644 --- a/elasticsearch-transport/test/unit/transport_manticore_test.rb +++ b/elasticsearch-transport/test/unit/transport_manticore_test.rb @@ -56,12 +56,14 @@ class Elasticsearch::Transport::Transport::HTTP::ManticoreTest < Test::Unit::Tes end should "set application/json header" do - @transport.connections.first.connection.stub("http://127.0.0.1:8080", body: '{"foo":"bar"}', headers: {"X-Content-Type" => "application/json"}, code: 200 ) - #@transport.connections.first.connection.expects(:get).returns(stub_everything) + options = { + :headers => { "content-type" => "application/json"} + } + transport = Manticore.new :hosts => [ { :host => 'localhost', :port => 8080 } ], :options => options + transport.connections.first.connection.stub("http://localhost:8080//", body: "\"\"", headers: {"content-type" => "application/json"}, code: 200 ) + response = transport.perform_request 'GET', '/', {} + assert_equal response.status, 200 - response = @transport.perform_request 'GET', '/', { :headers => {"X-Content-Type" => "application/json"} } - - assert_equal 'application/json', response.headers['content-type'] end should "handle HTTP methods" do @@ -77,17 +79,27 @@ class Elasticsearch::Transport::Transport::HTTP::ManticoreTest < Test::Unit::Tes end should "allow to set options for Manticore" do - transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234 } ] do |manticore| - manticore.headers["User-Agent"] = "myapp-0.0" - end + options = { :headers => {"User-Agent" => "myapp-0.0" }} + transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234 } ], :options => options + transport.connections.first.connection.expects(:get). + with('http://foobar:1234//', options).returns(stub_everything) - assert_equal "myapp-0.0", transport.connections.first.connection.headers["User-Agent"] + transport.perform_request 'GET', '/', {} end - should "set the credentials if passed" do - transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234, :user => 'foo', :password => 'bar' } ] - assert_equal 'foo', transport.connections.first.connection.username - assert_equal 'bar', transport.connections.first.connection.password + should "allow to set ssl options for Manticore" do + options = { + :ssl => { + :truststore => "test.jks", + :truststore_password => "test", + :verify => false + } + } + + ::Manticore::Client.expects(:new).with(:options => {:ignore_ssl_validation => true}) + transport = Manticore.new :hosts => [ { :host => 'foobar', :port => 1234 } ], :options => options + assert_equal java.lang.System.getProperty("javax.net.ssl.trustStore"), "test.jks" + assert_equal java.lang.System.getProperty("javax.net.ssl.trustStorePassword"), "test" end end