Skip to content

Commit

Permalink
Add search_scroll method
Browse files Browse the repository at this point in the history
This method is meant to replace `scan_scroll` in the future. It is,
however, not a drop-in replacement they have some subtle internal
differences.

Particularly, if you plan to upgrade Elasticsearch server from
pre-2.1.0 to anything newer than 2.1.0, you will need to replace
calls of `scan_scroll` method with `search_scroll`.

Elasticsearch has deprecated the search_type named `scan`, and
therefore the name `scan_scroll` also makes little sense.

See: https://www.elastic.co/guide/en/elasticsearch/reference/2.1/search-request-scroll.html
  • Loading branch information
gugod committed Jul 7, 2018
1 parent 2a8c8ab commit 3729d44
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 0 deletions.
31 changes: 31 additions & 0 deletions lib/Elastijk/oo.pm
Original file line number Diff line number Diff line change
Expand Up @@ -133,4 +133,35 @@ sub scan_scroll {
}
}

sub search_scroll {
my ($self, %args) = @_;
my $on_response_callback = delete $args{on_response};

my %uri_param = %{ delete($args{uri_param}) || {} };
$uri_param{scroll} ||= "10m";
my $scroll_id;
my ($status, $res) = $self->post(%args, command => "_search", uri_param => \%uri_param);
if (substr($status,0,1) ne '2') {
return;
}

my $r = $on_response_callback->($status, $res);
return if defined($r) && !$r;

while (1) {
$scroll_id = $res->{_scroll_id};
($status,$res) = $self->post(
path => "/_search/scroll",
body => { scroll => $uri_param{scroll}, scroll_id => $scroll_id }
);
if (substr($status,0,1) eq '2' && @{$res->{hits}{hits}} > 0) {
my $r = $on_response_callback->($status, $res);
last if defined($r) && !$r;
$scroll_id = $res->{_scroll_id};
} else {
last;
}
}
}

1;
7 changes: 7 additions & 0 deletions t/live-oo-scan-scroll.t
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ use Elastijk;

my $test_index_name = "test_index_$$".rand();
my $es = Elastijk->new(host => 'localhost', port => '9200', index => $test_index_name );

my $res = $es->get(path => "/");
if ($res->{version}{number} ge '2.1.0') {
plan skip_all => "scan_scroll does not make sense with Elasticsearch 2.1.0 or latter";
exit;
}

## create the index, and index some documents.
$es->put(
body => {
Expand Down
63 changes: 63 additions & 0 deletions t/live-oo-search-scroll.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env perl
use strict;
use warnings;
use Test::More;

unless ($ENV{TEST_LIVE}) {
plan skip_all => "Set env TEST_LIVE=1 to run this test."
}

use Elastijk;

my $test_index_name = "test_index_$$".rand();
my $es = Elastijk->new(host => 'localhost', port => '9200', index => $test_index_name );

my $res = $es->get(path => "/");
if ($res->{version}{number} lt '2.1.0') {
plan skip_all => "scan_scroll does not make sense with Elasticsearch older than 2.1.0";
exit;
}

## create the index, and index some documents.
$es->put(
body => {
settings => { index => {number_of_replicas => 0, number_of_shards => 1} },
mappings => { somedata => { properties => { somestr => { type => "string" }, someint => { type => "long" }}}},
aliases => { test_index => {} }
}
);
$es->post(command => "_refresh");

## create 500 documents
$es->post(
type => 'somestr',
body => {
# U+1F30x ~ U+1F56x
somestr => join("", map { chr(rand()*260+0x1F300) } (0..(10+rand()*128))),
someint => int(rand()*2**16),
}
) for (0..499);

$es->post(command => "_refresh");
sleep 2; # wait for refresh.
is $es->count(), 500, "count 500 documents";

## finally, testing search_scroll
my $count_callback = 0;
my $count = 0;
$es->search_scroll(
body => { size => 100, query => { match_all => {} } },
on_response => sub {
my ($status, $res) = @_;
$count += @{ $res->{hits}{hits} };
$count_callback++;
return 1;
}
);
is $count_callback, 5, "on_response is called exactly 5 times";
is $count, 500, "document count matches.";

## delete the index
$es->delete( index => $test_index_name );

done_testing;

0 comments on commit 3729d44

Please sign in to comment.