Skip to content
This repository
Browse code

[#449] Refactor & cleanup the new bulk support

* Use plain `Tire::Index#bulk` method name for the generic method:

    Tire.index('articles').bulk :index, [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ]

* Keep the `bulk_create`, `bulk_store`, `bulk_delete` methods

* Replace dynamically defined tests in index unit test with concrete examples
  • Loading branch information...
commit d2ed9802fef0a63a55c8e4d3d33667eee53a2398 1 parent 8e81c3a
Karel Minarik authored November 11, 2012
28  lib/tire/index.rb
@@ -89,8 +89,16 @@ def store(*args)
89 89
       curl = %Q|curl -X POST "#{url}" -d '#{document}'|
90 90
       logged([type, id].join('/'), curl)
91 91
     end
92  
-    
93  
-    def bulk_api_action(action, documents, options={})
  92
+
  93
+    def bulk(action, documents, options={})
  94
+      # TODO: A more Ruby-like DSL notation should be supported:
  95
+      #
  96
+      #     Tire.index('myindex').bulk do
  97
+      #       create id: 1, title: 'bar', _routing: 'abc'
  98
+      #       delete id: 1
  99
+      #       # ...
  100
+      #     end
  101
+      #
94 102
       payload = documents.map do |document|
95 103
         type = get_type_from_document(document, :escape => false) # Do not URL-escape the _type
96 104
         id   = get_id_from_document(document)
@@ -99,9 +107,7 @@ def bulk_api_action(action, documents, options={})
99 107
 
100 108
         output = []
101 109
         output << %Q|{"#{action}":{"_index":"#{@name}","_type":"#{type}","_id":"#{id}"}}|
102  
-        unless action == 'delete' # delete does not require a document, just an _id
103  
-          output << convert_document_to_json(document)
104  
-        end
  110
+        output << convert_document_to_json(document) unless action.to_s == 'delete'
105 111
         output.join("\n")
106 112
       end
107 113
       payload << ""
@@ -125,21 +131,21 @@ def bulk_api_action(action, documents, options={})
125 131
 
126 132
       ensure
127 133
         curl = %Q|curl -X POST "#{url}/_bulk" --data-binary '{... data omitted ...}'|
128  
-        logged('BULK', curl)
  134
+        logged('_bulk', curl)
129 135
       end
130  
-      
  136
+
131 137
     end
132  
-    
  138
+
133 139
     def bulk_create(documents, options={})
134  
-      bulk_api_action("create", documents, options)
  140
+      bulk :create, documents, options
135 141
     end
136 142
 
137 143
     def bulk_store(documents, options={})
138  
-      bulk_api_action("index", documents, options)
  144
+      bulk :index, documents, options
139 145
     end
140 146
 
141 147
     def bulk_delete(documents, options={})
142  
-      bulk_api_action("delete", documents, options)
  148
+      bulk :delete, documents, options
143 149
     end
144 150
 
145 151
     def import(klass_or_collection, options={})
150  test/unit/index_test.rb
@@ -491,7 +491,8 @@ class MyDocument;end; document = MyDocument.new
491 491
       end
492 492
 
493 493
       context "when performing a bulk api action" do
494  
-        # Possible api actions are index, create, delete
  494
+        # Possible Bulk API actions are `index`, `create`, `delete`
  495
+        #
495 496
         # The expected JSON looks like this:
496 497
         #
497 498
         # {"index":{"_index":"dummy","_type":"document","_id":"1"}}
@@ -501,59 +502,83 @@ class MyDocument;end; document = MyDocument.new
501 502
         # {"delete":{"_index":"dummy","_type":"document","_id":"2"}}
502 503
         #
503 504
         # See http://www.elasticsearch.org/guide/reference/api/bulk.html
504  
-        
505  
-        # test json with each type of action
506  
-        ["index","create","delete"].each do |action|
507  
-          should "serialize Hashes for #{action}" do
508  
-            Configuration.client.expects(:post).with do |url, json|
509  
-              
510  
-              url  == "#{@index.url}/_bulk" &&
511  
-              json =~ /"#{action}"/ &&
512  
-              json =~ /"_index":"dummy"/ &&
513  
-              json =~ /"_type":"document"/ &&
514  
-              json =~ /"_id":"1"/ &&
515  
-              json =~ /"_id":"2"/ &&
516  
-              if action == 'delete'
517  
-                # delete should not contain the document contents
518  
-                true
519  
-              else              
520  
-                json =~ /"id":"1"/ &&
521  
-                json =~ /"id":"2"/ &&
522  
-                json =~ /"title":"One"/ &&
523  
-                json =~ /"title":"Two"/
524  
-              end
525  
-            end.returns(mock_response('{}'), 200)
526  
-          
527  
-            @index.bulk_api_action action,  [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ]
528  
-          end
529 505
 
530  
-          should "serialize ActiveModel instances for #{action}" do
531  
-            Configuration.client.expects(:post).with do |url, json|
532  
-              url  == "#{ActiveModelArticle.index.url}/_bulk" &&
533  
-              json =~ /"#{action}"/ &&
534  
-              json =~ /"_index":"active_model_articles"/ &&
535  
-              json =~ /"_type":"active_model_article"/ &&
536  
-              json =~ /"_id":"1"/ &&
537  
-              json =~ /"_id":"2"/ &&
538  
-              if action == 'delete'
539  
-                # delete should not contain the document contents
540  
-                true
541  
-              else              
542  
-                json =~ /"title":"One"/ &&
543  
-                json =~ /"title":"Two"/
544  
-              end
545  
-              
546  
-            end.returns(mock_response('{}', 200))
547  
-          
548  
-            one = ActiveModelArticle.new 'title' => 'One'; one.id = '1'
549  
-            two = ActiveModelArticle.new 'title' => 'Two'; two.id = '2'
550  
-          
551  
-            ActiveModelArticle.index.bulk_api_action action, [ one, two ]
552  
-          end
553  
-          
  506
+        should "serialize payload for index action" do
  507
+          Configuration.client.
  508
+            expects(:post).
  509
+            with do |url, payload|
  510
+              assert_equal "#{@index.url}/_bulk", url
  511
+              assert_match /"index"/, payload
  512
+              assert_match /"_index":"dummy"/, payload
  513
+              assert_match /"_type":"document"/, payload
  514
+              assert_match /"_id":"1"/, payload
  515
+              assert_match /"_id":"2"/, payload
  516
+              assert_match /"title":"One"/, payload
  517
+              assert_match /"title":"Two"/, payload
  518
+            end.
  519
+            returns(mock_response('{}'), 200)
  520
+
  521
+          @index.bulk :index, [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ]
  522
+        end
  523
+
  524
+        should "serialize payload for create action" do
  525
+          Configuration.client.
  526
+            expects(:post).
  527
+            with do |url, payload|
  528
+              assert_equal "#{@index.url}/_bulk", url
  529
+              assert_match /"create"/, payload
  530
+              assert_match /"_index":"dummy"/, payload
  531
+              assert_match /"_type":"document"/, payload
  532
+              assert_match /"_id":"1"/, payload
  533
+              assert_match /"_id":"2"/, payload
  534
+              assert_match /"title":"One"/, payload
  535
+              assert_match /"title":"Two"/, payload
  536
+            end.
  537
+            returns(mock_response('{}'), 200)
  538
+
  539
+          @index.bulk :create, [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ]
554 540
         end
555  
-        # use the index action to test common features of the bulk api
556  
-        context "namespaced models" do
  541
+
  542
+        should "serialize payload for delete action" do
  543
+          Configuration.client.
  544
+            expects(:post).
  545
+            with do |url, payload|
  546
+              assert_equal "#{@index.url}/_bulk", url
  547
+              assert_match /"delete"/, payload
  548
+              assert_match /"_index":"dummy"/, payload
  549
+              assert_match /"_type":"document"/, payload
  550
+              assert_match /"_id":"1"/, payload
  551
+              assert_match /"_id":"2"/, payload
  552
+              assert ! payload.include?('"title"')
  553
+            end.
  554
+            returns(mock_response('{}'), 200)
  555
+
  556
+          @index.bulk :delete, [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ]
  557
+        end
  558
+
  559
+        should "serialize ActiveModel instances as payload" do
  560
+          Configuration.client.
  561
+            expects(:post).
  562
+            with do |url, payload|
  563
+              assert_equal "#{ActiveModelArticle.index.url}/_bulk", url
  564
+              assert_match /"index"/, payload
  565
+              assert_match /"_index":"active_model_articles"/, payload
  566
+              assert_match /"_type":"active_model_article"/, payload
  567
+              assert_match /"_id":"1"/, payload
  568
+              assert_match /"_id":"2"/, payload
  569
+              assert_match /"title":"One"/, payload
  570
+              assert_match /"title":"Two"/, payload
  571
+            end.
  572
+            returns(mock_response('{}'), 200)
  573
+
  574
+          one = ActiveModelArticle.new 'title' => 'One'; one.id = '1'
  575
+          two = ActiveModelArticle.new 'title' => 'Two'; two.id = '2'
  576
+
  577
+          ActiveModelArticle.index.bulk :index, [ one, two ]
  578
+        end
  579
+
  580
+        context "with namespaced models" do
  581
+
557 582
           should "not URL-escape the document_type" do
558 583
             Configuration.client.expects(:post).with do |url, json|
559 584
               # puts url, json
@@ -561,42 +586,45 @@ class MyDocument;end; document = MyDocument.new
561 586
               json =~ %r|"_index":"my_namespace_my_models"| &&
562 587
               json =~ %r|"_type":"my_namespace/my_model"|
563 588
             end.returns(mock_response('{}', 200))
564  
-        
  589
+
565 590
             module MyNamespace
566 591
               class MyModel
567 592
                 def document_type;   "my_namespace/my_model"; end
568 593
                 def to_indexed_json; "{}";                    end
569 594
               end
570 595
             end
571  
-        
572  
-            Tire.index('my_namespace_my_models').bulk_api_action "index", [ MyNamespace::MyModel.new ]
  596
+
  597
+            Tire.index('my_namespace_my_models').bulk :index, [ MyNamespace::MyModel.new ]
573 598
           end
574 599
         end
  600
+
575 601
         should "try again when an exception occurs" do
576 602
           Configuration.client.expects(:post).returns(mock_response('Server error', 503)).at_least(2)
577 603
 
578  
-          assert !@index.bulk_api_action("index", [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ])
  604
+          assert !@index.bulk(:index, [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ])
579 605
         end
580 606
 
581 607
         should "try again and the raise when an exception occurs" do
582 608
           Configuration.client.expects(:post).returns(mock_response('Server error', 503)).at_least(2)
583 609
 
584 610
           assert_raise(RuntimeError) do
585  
-            @index.bulk_api_action("index", [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ], {:raise => true})
  611
+            @index.bulk :index,
  612
+                        [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ],
  613
+                        :raise => true
586 614
           end
587 615
         end
588 616
 
589 617
         should "try again when a connection error occurs" do
590 618
           Configuration.client.expects(:post).raises(Errno::ECONNREFUSED, "Connection refused - connect(2)").at_least(2)
591 619
 
592  
-          assert !@index.bulk_api_action("index", [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ])
  620
+          assert !@index.bulk(:index, [ {:id => '1', :title => 'One'} ])
593 621
         end
594 622
 
595 623
         should "retry on SIGINT type of exceptions" do
596 624
           Configuration.client.expects(:post).raises(Interrupt, "abort then interrupt!")
597 625
 
598 626
           assert_raise Interrupt do
599  
-            @index.bulk_api_action("index", [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ])
  627
+            @index.bulk :index, [ {:id => '1', :title => 'One'} ]
600 628
           end
601 629
         end
602 630
 
@@ -608,7 +636,7 @@ def to_indexed_json; "{}";                    end
608 636
           STDERR.expects(:puts).once
609 637
 
610 638
           documents = [ { :title => 'Bogus' }, { :title => 'Real', :id => 1 } ]
611  
-          ActiveModelArticle.index.bulk_api_action("index", documents)
  639
+          ActiveModelArticle.index.bulk :index, documents
612 640
         end
613 641
 
614 642
         should "log the response code" do
@@ -619,7 +647,7 @@ def to_indexed_json; "{}";                    end
619 647
             status == 200
620 648
           end
621 649
 
622  
-          @index.bulk_api_action("index",  [ {:id => '1', :title => 'One'}, {:id => '2', :title => 'Two'} ])
  650
+          @index.bulk :index,  [ {:id => '1', :title => 'One'} ]
623 651
         end
624 652
 
625 653
       end

0 notes on commit d2ed980

Please sign in to comment.
Something went wrong with that request. Please try again.