properly detect the end-of-scroll condition.

gugod · May 28, 2014 · 3993117 · 3993117
1 parent e855824
commit 3993117
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 14 deletions.
diff --git a/lib/Elastijk.pm b/lib/Elastijk.pm
@@ -39,7 +39,6 @@ sub request {
 
 sub request_raw {
     my $args = _build_hijk_request_args($_[0]);
-    print Data::Dumper::Dumper($args) if $args->{method} eq "GET";
     my $res = Hijk::request($args);
     return $res->{status}, $res->{body};
 }

diff --git a/lib/Elastijk/oo.pm b/lib/Elastijk/oo.pm
@@ -87,20 +87,24 @@ sub scan_scroll {
 
     my %uri_param = %{ delete($args{uri_param}) || {} };
     $uri_param{search_type} = "scan";
-    $uri_param{scroll} //= "10m";
-    my ($status, $res) = $self->get(%args, command => "_search", uri_param => \%uri_param);
+    $uri_param{scroll} ||= "10m";
     my $scroll_id;
+    my ($status, $res) = $self->get(%args, command => "_search", uri_param => \%uri_param);
+    if (substr($status,0,1) ne '2') {
+        return;
+    }
 
-    while (substr($status,0,1) eq '2') {
+    while (1) {
         $scroll_id = $res->{_scroll_id};
         ($status,$res) = $self->get(
             index => "_search", type => "scroll", #WTF
             uri_param => { scroll => $uri_param{scroll}, scroll_id => $scroll_id }
         );
-        last unless substr($status,0,1) eq '2';
-        print ">> $status, $res->{_scroll_id}\n";
-        my $r = $on_response_callback->($status, $res);
-        if (defined($r) && !$r) {
+        if (substr($status,0,1) eq '2' && @{$res->{hits}{hits}} > 0) {
+            my $r = $on_response_callback->($status, $res);
+            last if defined($r) && !$r;
+            $scroll_id = $res->{_scroll_id};
+        } else {
             last;
         }
     }

diff --git a/t/live-oo-scan-scroll.t b/t/live-oo-scan-scroll.t
@@ -15,7 +15,8 @@ my $es = Elastijk->new(host => 'localhost', port => '9200', index => $test_index
 $es->put(
     body => {
         settings => { index => {number_of_replicas => 0, number_of_shards => 1} },
-        mappings => { somedata => { properties => { somestr => { type => "string" }, someint => { type => "long" }}}}
+        mappings => { somedata => { properties => { somestr => { type => "string" }, someint => { type => "long" }}}},
+        aliases => { test_index => {} }
     }
 );
 
@@ -27,21 +28,25 @@ $es->post(
         somestr => join("", map { chr(rand()*260+0x1F300) } (0..(10+rand()*128))),
         someint => int(rand()*2**16),
     }
-) for (0..4999);
+) for (0..499);
 
 sleep 2; # wait for refresh.
-is $es->count(), 5000, "count 5000 documents";
+is $es->count(), 500, "count 500 documents";
 
 ## finally, testing scan_scroll
+my $count_callback = 0;
 my $count = 0;
 $es->scan_scroll(
-    body => { size => 1000, query => { match_all => {} } },
+    body => { size => 100, query => { match_all => {} } },
     on_response => sub {
-        $count++;
+        my ($status, $res) = @_;
+        $count += @{ $res->{hits}{hits} };
+        $count_callback++;
         return 1;
     }
 );
-is $count, 5, "on_response is called exactly 5 times.";
+is $count_callback, 5, "on_response is called exactly 5 times";
+is $count, 500, "document count matches.";
 
 ## delete the index
 $es->delete( index => $test_index_name );