Skip to content
This repository has been archived by the owner on Jun 30, 2018. It is now read-only.

Commit

Permalink
Added support for reindexing the index documents into new index
Browse files Browse the repository at this point in the history
Added a `Tire::Index#reindex` method which uses the "scan/scroll" API to retrieve
documents from index A and import them into index B.

If the new index does not exists, it is created: you may pass settings as an options hash.

You may pass a Tire DSL query as a block to the method.

Example:
--------

    require 'tire'

    Tire.configure { logger STDERR }

    Tire.index('scantest-new') { delete }

    documents = (1..100).map do |i|
      { id: i, type: 'test', title: "Document #{i}" }
    end

    puts "", "Importing data into ElasticSearch..."

    Tire.index 'scantest' do
      delete and create :settings => { :number_of_shards => 1, :number_of_replicas => 0 }
      import documents
      refresh
    end

    Tire.index('scantest').reindex 'scantest-new', settings: { number_of_shards: 3 } do
      query { string '10*' }
    end
  • Loading branch information
karmi committed Mar 30, 2012
1 parent c154474 commit 88b5bc7
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 0 deletions.
1 change: 1 addition & 0 deletions lib/tire.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

require 'active_support/core_ext/object/to_param'
require 'active_support/core_ext/object/to_query'
require 'active_support/core_ext/hash/except.rb'

# Ruby 1.8 compatibility
require 'tire/rubyext/ruby_1_8' if defined?(RUBY_VERSION) && RUBY_VERSION < '1.9'
Expand Down
11 changes: 11 additions & 0 deletions lib/tire/index.rb
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,17 @@ def import(klass_or_collection, options={})
end
end

def reindex(name, options={}, &block)
new_index = Index.new(name)
new_index.create(options) unless new_index.exists?

Search::Scan.new(self.name, &block).each do |results|
new_index.bulk_store results.map do |document|
document.to_hash.except(:type, :_index, :_explanation, :_score, :_version, :highlight, :sort)
end
end
end

def remove(*args)
if args.size > 1
type, document = args
Expand Down
46 changes: 46 additions & 0 deletions test/integration/reindex_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
require 'test_helper'

module Tire

class ReindexIntegrationTest < Test::Unit::TestCase
include Test::Integration

context "Reindex" do
setup do
Tire.index('reindex-test-new').delete

documents = (1..100).map { |i| { id: i, type: 'test', title: "Document #{i}" } }

Tire.index 'reindex-test' do
delete
create :settings => { :number_of_shards => 1, :number_of_replicas => 0 }
import documents
refresh
end
end

teardown do
Index.new('reindex-test').delete
Index.new('reindex-test-new').delete
end

should "reindex the index into a new index with different settings" do
Tire.index('reindex-test').reindex 'reindex-test-new', settings: { number_of_shards: 3 }

Tire.index('reindex-test-new').refresh
assert_equal 100, Tire.search('reindex-test-new').results.total
assert_equal '3', Tire.index('reindex-test-new').settings['index.number_of_shards']
end

should "reindex a portion of an index into a new index" do
Tire.index('reindex-test').reindex('reindex-test-new') { query { string '10*' } }

Tire.index('reindex-test-new').refresh
assert_equal 2, Tire.search('reindex-test-new').results.total
end

end

end

end
47 changes: 47 additions & 0 deletions test/unit/index_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -720,6 +720,53 @@ def self.count; DATA.size; end

end

context "reindexing" do
setup do
@results = {
"_scroll_id" => "abc123",
"took" => 3,
"hits" => {
"total" => 10,
"hits" => [
{ "_id" => "1", "_source" => { "title" => "Test" } }
]
}
}
end

should "perform bulk store in the new index" do
Index.any_instance.stubs(:exists?).returns(true)
Search::Scan.any_instance.stubs(:__perform)
Search::Scan.any_instance
.expects(:results)
.returns(Results::Collection.new(@results))
.then
.returns(Results::Collection.new(@results.merge('hits' => {'hits' => []})))
.at_least_once

Index.any_instance.expects(:bulk_store).once

@index.reindex 'whammy'
end

should "create the new index if it does not exist" do
options = { :settings => { :number_of_shards => 1 } }

Index.any_instance.stubs(:exists?).returns(false)
Search::Scan.any_instance.stubs(:__perform)
Search::Scan.any_instance
.expects(:results)
.returns(Results::Collection.new(@results))
.then
.returns(Results::Collection.new(@results.merge('hits' => {'hits' => []})))
.at_least_once

Index.any_instance.expects(:create).with(options).once

@index.reindex 'whammy', options
end

end
end

end
Expand Down

0 comments on commit 88b5bc7

Please sign in to comment.