Permalink
Browse files

Merge remote-tracking branch 'upstream/master'

* upstream/master: (39 commits)
  Release 0.4.0
  [GEMS] Relaxed gem dependencies for "rest-client", "multi_json" and "bundler"
  Release 0.4.0.rc
  [#218] Cleaned up the test suite for document_type in Index#bulk_store
  [#218] Fixed the incorrect serialization of `document_type` in Index#bulk_store
  [FIX] Fixed displaying of Rake task usage
  [#289] Update README and documentation, use the `prefix` query as the example of "unsupported" query
  [#289] Added an example of fuzzy query in the Text query integration test
  [TEST] Improved code in the Text Query integration test
  [TEST] Improved code legibility in the Explanation integration test
  [TEST] Improved code legibility in the DSL integration test
  [#289] Cleaned up the code for "term" and "fuzzy" queries and restructured the search query test suite
  [#289] Improvements to the "term" query type
  [#289] Added the "fuzzy" query type
  [GEMS] Updated and cleaned up gem dependencies
  Restructured the test suite for Ruby 1.8 unit test compatibility
  Added Ruby 1.8 compatibility for `Utils.escape/unescape`
  [TEST] Removed the "supermodel" gem and used the "redis-persistence" gem
  [ACTIVEMODEL] Fixed and cleaned up URL-escaping of document type
  [UTILS] Added the `Tire::Utils` module
  ...
  • Loading branch information...
Evan-M committed Mar 28, 2012
2 parents 42f5786 + 8ab5774 commit 7eb34b0819bbd65b7f6ae9d66f830ef569bcc5a8
Showing with 865 additions and 231 deletions.
  1. +1 −1 README.markdown
  2. +1 −0 examples/tire-dsl.rb
  3. +8 −0 lib/tire.rb
  4. +7 −6 lib/tire/dsl.rb
  5. +18 −11 lib/tire/index.rb
  6. +3 −2 lib/tire/model/import.rb
  7. +2 −1 lib/tire/model/naming.rb
  8. +1 −1 lib/tire/model/persistence.rb
  9. +20 −17 lib/tire/results/collection.rb
  10. +1 −1 lib/tire/results/item.rb
  11. +7 −0 lib/tire/rubyext/ruby_1_8.rb
  12. +33 −19 lib/tire/search.rb
  13. +5 −0 lib/tire/search/facet.rb
  14. +8 −3 lib/tire/search/query.rb
  15. +47 −14 lib/tire/tasks.rb
  16. +17 −0 lib/tire/utils.rb
  17. +15 −4 lib/tire/version.rb
  18. +1 −0 test/integration/active_model_indexing_test.rb
  19. +7 −5 test/integration/active_model_searchable_test.rb
  20. +159 −72 test/integration/active_record_searchable_test.rb
  21. +34 −0 test/integration/count_test.rb
  22. +22 −0 test/integration/dsl_search_test.rb
  23. +44 −0 test/integration/explanation_test.rb
  24. +15 −0 test/integration/facets_test.rb
  25. +20 −0 test/integration/fuzzy_queries_test.rb
  26. +1 −0 test/integration/mongoid_searchable_test.rb
  27. +22 −1 test/integration/persistent_model_test.rb
  28. +17 −3 test/integration/text_query_test.rb
  29. +43 −1 test/models/active_record_models.rb
  30. +0 −1 test/models/mongoid_models.rb
  31. +12 −0 test/models/persistent_article_in_namespace.rb
  32. +5 −10 test/models/supermodel_article.rb
  33. +16 −2 test/test_helper.rb
  34. +86 −16 test/unit/index_test.rb
  35. +4 −4 test/unit/model_import_test.rb
  36. +1 −9 test/unit/model_search_test.rb
  37. +6 −0 test/unit/results_collection_test.rb
  38. +8 −0 test/unit/results_item_test.rb
  39. +9 −0 test/unit/search_facet_test.rb
  40. +36 −7 test/unit/search_query_test.rb
  41. +70 −1 test/unit/search_test.rb
  42. +23 −12 test/unit/tire_test.rb
  43. +10 −7 tire.gemspec
View
@@ -295,7 +295,7 @@ If configuring the search payload with blocks feels somehow too weak for you, yo
a plain old Ruby `Hash` (or JSON string) with the query declaration to the `search` method:
```ruby
- Tire.search 'articles', :query => { :fuzzy => { :title => 'Sour' } }
+ Tire.search 'articles', :query => { :prefix => { :title => 'fou' } }
```
If this sounds like a great idea to you, you are probably able to write your application
View
@@ -481,6 +481,7 @@ def self.search
# * [terms](http://elasticsearch.org/guide/reference/query-dsl/terms-query.html)
# * [bool](http://www.elasticsearch.org/guide/reference/query-dsl/bool-query.html)
# * [custom_score](http://www.elasticsearch.org/guide/reference/query-dsl/custom-score-query.html)
+# * [fuzzy](http://www.elasticsearch.org/guide/reference/query-dsl/fuzzy-query.html)
# * [all](http://www.elasticsearch.org/guide/reference/query-dsl/match-all-query.html)
# * [ids](http://www.elasticsearch.org/guide/reference/query-dsl/ids-query.html)
View
@@ -2,9 +2,17 @@
require 'multi_json'
require 'active_model'
require 'hashr'
+require 'cgi'
+
+require 'active_support/core_ext/object/to_param'
+require 'active_support/core_ext/object/to_query'
+
+# Ruby 1.8 compatibility
+require 'tire/rubyext/ruby_1_8' if defined?(RUBY_VERSION) && RUBY_VERSION < '1.9'
require 'tire/rubyext/hash'
require 'tire/rubyext/symbol'
+require 'tire/utils'
require 'tire/logger'
require 'tire/configuration'
require 'tire/http/response'
View
@@ -10,15 +10,16 @@ def search(indices=nil, options={}, &block)
Search::Search.new(indices, options, &block)
else
payload = case options
- when Hash then options.to_json
- when String then options
+ when Hash then
+ options
+ when String then
+ Tire.warn "Passing the payload as a JSON string in Tire.search has been deprecated, " +
+ "please use the block syntax or pass a plain Hash."
+ options
else raise ArgumentError, "Please pass a Ruby Hash or String with JSON"
end
- response = Configuration.client.post( "#{Configuration.url}/#{indices}/_search", payload)
- raise Tire::Search::SearchRequestFailed, response.to_s if response.failure?
- json = MultiJson.decode(response.body)
- results = Results::Collection.new(json, options)
+ Search::Search.new(indices, :payload => payload)
end
rescue Exception => error
STDERR.puts "[REQUEST FAILED] #{error.class} #{error.message rescue nil}\n"
View
@@ -1,7 +1,7 @@
module Tire
class Index
- attr_reader :name
+ attr_reader :name, :response
def initialize(name, &block)
@name = name
@@ -64,10 +64,10 @@ def store(*args)
logged([type, id].join('/'), curl)
end
- def bulk_store documents
+ def bulk_store(documents, options={})
payload = documents.map do |document|
+ type = get_type_from_document(document, :escape => false) # Do not URL-escape the _type
id = get_id_from_document(document)
- type = get_type_from_document(document)
STDERR.puts "[ERROR] Document #{document.inspect} does not have ID" unless id
@@ -92,6 +92,7 @@ def bulk_store documents
retry
else
STDERR.puts "[ERROR] Too many exceptions occured, giving up. The HTTP response was: #{error.message}"
+ raise if options[:raise]
end
ensure
@@ -100,32 +101,33 @@ def bulk_store documents
end
end
- def import(klass_or_collection, method=nil, options={})
+ def import(klass_or_collection, options={})
case
- when method
+ when method = options.delete(:method)
options = {:page => 1, :per_page => 1000}.merge options
while documents = klass_or_collection.send(method.to_sym, options.merge(:page => options[:page])) \
and documents.to_a.length > 0
documents = yield documents if block_given?
- bulk_store documents
+ bulk_store documents, options
options[:page] += 1
end
when klass_or_collection.respond_to?(:map)
documents = block_given? ? yield(klass_or_collection) : klass_or_collection
- bulk_store documents
+ bulk_store documents, options
else
- raise ArgumentError, "Please pass either a collection of objects, " +
- "or method for fetching records, or Enumerable compatible class"
+ raise ArgumentError, "Please pass either an Enumerable compatible class, or a collection object" +
+ "with a method for fetching records in batches (such as 'paginate')"
end
end
def remove(*args)
if args.size > 1
type, document = args
+ type = Utils.escape(type)
id = get_id_from_document(document) || document
else
document = args.pop
@@ -146,6 +148,7 @@ def remove(*args)
def retrieve(type, id)
raise ArgumentError, "Please pass a document ID" unless id
+ type = Utils.escape(type)
url = "#{Configuration.url}/#{@name}/#{type}/#{id}"
@response = Configuration.client.get url
@@ -262,7 +265,9 @@ def logged(endpoint='/', curl='')
end
end
- def get_type_from_document(document)
+ def get_type_from_document(document, options={})
+ options = {:escape => true}.merge(options)
+
old_verbose, $VERBOSE = $VERBOSE, nil # Silence Object#type deprecation warnings
type = case
when document.respond_to?(:document_type)
@@ -275,7 +280,9 @@ def get_type_from_document(document)
document.type
end
$VERBOSE = old_verbose
- type || :document
+
+ type ||= 'document'
+ options[:escape] ? Utils.escape(type) : type
end
def get_id_from_document(document)
View
@@ -1,3 +1,4 @@
+
module Tire
module Model
@@ -13,8 +14,8 @@ module Import
module ClassMethods
def import options={}, &block
- method = options.delete(:method) || 'paginate'
- index.import klass, method, options, &block
+ options = { :method => 'paginate' }.update options
+ index.import klass, options, &block
end
end
View
@@ -35,6 +35,7 @@ def index_name name=nil, &block
@index_name = name if name
@index_name = block if block_given?
@index_name = Tire::Configuration.global_index_name if Tire::Configuration.global_index_name
+ # TODO: Try to get index_name from ancestor classes
@index_name || [index_prefix, klass.model_name.plural].compact.join('_')
end
@@ -79,7 +80,7 @@ def index_prefix(*args)
#
def document_type name=nil
@document_type = name if name
- @document_type || klass.model_name.singular
+ @document_type || klass.model_name.underscore
end
end
@@ -45,7 +45,7 @@ def self.included(base)
include Persistence::Storage
- ['_score', '_type', '_index', '_version', 'sort', 'highlight', 'matches'].each do |attr|
+ ['_score', '_type', '_index', '_version', 'sort', 'highlight', 'matches', '_explanation'].each do |attr|
define_method("#{attr}=") { |value| @attributes ||= {}; @attributes[attr] = value }
define_method("#{attr}") { @attributes[attr] }
end
@@ -18,7 +18,8 @@ def initialize(response, options={})
def results
@results ||= begin
- hits = @response['hits']['hits']
+ hits = @response['hits']['hits'].map { |d| d.update '_type' => Utils.unescape(d['_type']) }
+
unless @options[:load]
if @wrapper == Hash
hits
@@ -31,33 +32,35 @@ def results
document.update( {'id' => h['_id']} )
# Update the document with meta information
- ['_score', '_type', '_index', '_version', 'sort', 'highlight'].each { |key| document.update( {key => h[key]} || {} ) }
+ ['_score', '_type', '_index', '_version', 'sort', 'highlight', '_explanation'].each { |key| document.update( {key => h[key]} || {} ) }
# Return an instance of the "wrapper" class
@wrapper.new(document)
end
end
+
else
return [] if hits.empty?
- type = @response['hits']['hits'].first['_type']
- raise NoMethodError, "You have tried to eager load the model instances, " +
- "but Tire cannot find the model class because " +
- "document has no _type property." unless type
-
- begin
- klass = type.camelize.constantize
- rescue NameError => e
- raise NameError, "You have tried to eager load the model instances, but " +
- "Tire cannot find the model class '#{type.camelize}' " +
- "based on _type '#{type}'.", e.backtrace
+ records = {}
+ @response['hits']['hits'].group_by { |item| item['_type'] }.each do |type, items|
+ raise NoMethodError, "You have tried to eager load the model instances, " +
+ "but Tire cannot find the model class because " +
+ "document has no _type property." unless type
+
+ begin
+ klass = type.camelize.constantize
+ rescue NameError => e
+ raise NameError, "You have tried to eager load the model instances, but " +
+ "Tire cannot find the model class '#{type.camelize}' " +
+ "based on _type '#{type}'.", e.backtrace
+ end
+ ids = items.map { |h| h['_id'] }
+ records[type] = @options[:load] === true ? klass.find(ids) : klass.find(ids, @options[:load])
end
- ids = @response['hits']['hits'].map { |h| h['_id'] }
- records = @options[:load] === true ? klass.find(ids) : klass.find(ids, @options[:load])
-
# Reorder records to preserve order from search results
- ids.map { |id| records.detect { |record| record.id.to_s == id.to_s } }
+ @response['hits']['hits'].map { |item| records[item['_type']].detect { |record| record.id.to_s == item['_id'].to_s } }
end
end
end
View
@@ -28,7 +28,7 @@ def method_missing(method_name, *arguments)
end
def [](key)
- @attributes[key]
+ @attributes[key.to_sym]
end
def id
@@ -0,0 +1,7 @@
+require 'rubygems'
+
+# Require URI escape/unescape compatibility layer from Rack
+#
+# See <http://www.ruby-doc.org/stdlib-1.9.3/libdoc/uri/rdoc/URI.html#method-c-encode_www_form_component>
+#
+require 'rack/backports/uri/common_18'
View
@@ -1,14 +1,14 @@
module Tire
module Search
class SearchRequestFailed < StandardError; end
-
+
class Search
- attr_reader :indices, :json, :query, :facets, :filters, :options
+ attr_reader :indices, :json, :query, :facets, :filters, :options, :explain
- def initialize(indices=nil, options = {}, &block)
+ def initialize(indices=nil, options={}, &block)
@indices = Array(indices)
- @types = Array(options.delete(:type))
+ @types = Array(options.delete(:type)).map { |type| Utils.escape(type) }
@options = options
@path = ['/', @indices.join(','), @types.join(','), '_search'].compact.join('/').squeeze('/')
@@ -28,6 +28,10 @@ def url
Configuration.url + @path
end
+ def params
+ @options.empty? ? '' : '?' + @options.to_param
+ end
+
def query(&block)
@query = Query.new
block.arity < 1 ? @query.instance_eval(&block) : block.call(@query)
@@ -77,12 +81,17 @@ def fields(*fields)
self
end
+ def explain(value)
+ @explain = value
+ self
+ end
+
def version(value)
@version = value
end
def perform
- @response = Configuration.client.get(self.url, self.to_json)
+ @response = Configuration.client.get(self.url + self.params, self.to_json)
if @response.failure?
STDERR.puts "[REQUEST FAILED] #{self.to_curl}\n"
raise SearchRequestFailed, @response.to_s
@@ -95,26 +104,31 @@ def perform
end
def to_curl
- %Q|curl -X GET "#{self.url}?pretty=true" -d '#{self.to_json}'|
+ %Q|curl -X GET "#{url}#{params.empty? ? '?' : params.to_s + '&'}pretty=true" -d '#{to_json}'|
end
def to_hash
- request = {}
- request.update( { :query => @query.to_hash } ) if @query
- request.update( { :sort => @sort.to_ary } ) if @sort
- request.update( { :facets => @facets.to_hash } ) if @facets
- request.update( { :filter => @filters.first.to_hash } ) if @filters && @filters.size == 1
- request.update( { :filter => { :and => @filters.map { |filter| filter.to_hash } } } ) if @filters && @filters.size > 1
- request.update( { :highlight => @highlight.to_hash } ) if @highlight
- request.update( { :size => @size } ) if @size
- request.update( { :from => @from } ) if @from
- request.update( { :fields => @fields } ) if @fields
- request.update( { :version => @version } ) if @version
- request
+ @options.delete(:payload) || begin
+ request = {}
+ request.update( { :query => @query.to_hash } ) if @query
+ request.update( { :sort => @sort.to_ary } ) if @sort
+ request.update( { :facets => @facets.to_hash } ) if @facets
+ request.update( { :filter => @filters.first.to_hash } ) if @filters && @filters.size == 1
+ request.update( { :filter => { :and => @filters.map {|filter| filter.to_hash} } } ) if @filters && @filters.size > 1
+ request.update( { :highlight => @highlight.to_hash } ) if @highlight
+ request.update( { :size => @size } ) if @size
+ request.update( { :from => @from } ) if @from
+ request.update( { :fields => @fields } ) if @fields
+ request.update( { :version => @version } ) if @version
+ request.update( { :explain => @explain } ) if @explain
+ request
+ end
end
def to_json
- to_hash.to_json
+ payload = to_hash
+ # TODO: Remove when deprecated interface is removed
+ payload.is_a?(String) ? payload : payload.to_json
end
def logged(error=nil)
View
@@ -51,6 +51,11 @@ def query(&block)
@value = { :query => Query.new(&block).to_hash }
end
+ def filter(field, value, options={})
+ @value = { :filter => { :term => { field => value }}.update(options) }
+ self
+ end
+
def to_json
to_hash.to_json
end
Oops, something went wrong.

0 comments on commit 7eb34b0

Please sign in to comment.