Conversion to Ref::Util and Types::Tiny

Fix paths for the shebang line so the installer will be able to install the scripts with the correct Perl binary path. Add a first pass at nested query syntax
reyjrar · Feb 22, 2018 · 1dcaaa7 · 1dcaaa7
1 parent 722e812
commit 1dcaaa7
Show file tree

Hide file tree

Showing 26 changed files with 243 additions and 127 deletions.
diff --git a/CopyIndexes.mkdn b/CopyIndexes.mkdn
@@ -4,7 +4,7 @@ es-copy-index.pl - Copy an index from one cluster to another
 
 # VERSION
 
-version 5.4
+version 5.5
 
 # SYNOPSIS
 

diff --git a/Maintenance.mkdn b/Maintenance.mkdn
@@ -4,7 +4,7 @@ es-daily-index-maintenance.pl - Run to prune old indexes and optimize existing
 
 # VERSION
 
-version 5.4
+version 5.5
 
 # SYNOPSIS
 

diff --git a/README.mkdn b/README.mkdn
@@ -4,7 +4,7 @@ App::ElasticSearch::Utilities - Utilities for Monitoring ElasticSearch
 
 # VERSION
 
-version 5.4
+version 5.5
 
 # SYNOPSIS
 

diff --git a/Searching.mkdn b/Searching.mkdn
@@ -4,7 +4,7 @@ es-search.pl - Provides a CLI for quick searches of data in ElasticSearch daily
 
 # VERSION
 
-version 5.4
+version 5.5
 
 # SYNOPSIS
 
@@ -87,8 +87,8 @@ Examples might include:
     es-search.pl program:"apache" AND crit:500 --show file,out_bytes
 
     # Search for ip subnet client IP 1.2.3.0 to 1.2.3.255 or 1.2.0.0 to 1.2.255.255
-    es-search.pl --size=100 dst:"admin.example.com" AND src_ip:"1.2.3.*"
-    es-search.pl --size=100 dst:"admin.example.com" AND src_ip:"1.2.*"
+    es-search.pl --size=100 dst:"admin.example.com" AND src_ip:"1.2.3.0/24"
+    es-search.pl --size=100 dst:"admin.example.com" AND src_ip:"1.2.0/16"
 
     # Show the top src_ip for 'www.example.com'
     es-search.pl --base access dst:www.example.com --top src_ip

diff --git a/dist.ini b/dist.ini
@@ -37,7 +37,6 @@ copy = README.mkdn
 [MetaJSON]
 [MetaConfig]
 [MinimumPerl]
-perl = 5.010001
 
 ;Changelog
 [ChangelogFromGit]

diff --git a/lib/App/ElasticSearch/Utilities.pm b/lib/App/ElasticSearch/Utilities.pm
@@ -528,7 +528,7 @@ sub es_connect {
 
     # If we're overriding, return a unique handle
     if(defined $override_servers) {
-        my @overrides = ref $override_servers eq 'ARRAY' ? @$override_servers : $override_servers;
+        my @overrides =  is_arrayref($override_servers) ? @$override_servers : $override_servers;
         my @servers;
         foreach my $entry ( @overrides ) {
             my ($s,$p) = split /\:/, $entry;
@@ -613,7 +613,7 @@ sub es_request {
         else {
             # Validate each included index
             my @valid;
-            my @test = ref $index_in eq 'ARRAY' ? @{ $index_in } : split /\,/, $index_in;
+            my @test = is_arrayref($index_in) ? @{ $index_in } : split /\,/, $index_in;
             foreach my $i (@test) {
                 push @valid, $i if es_index_valid($i);
             }
@@ -1007,7 +1007,7 @@ sub _find_fields {
     elsif( exists $ref->{type} ) {
         _add_fields($f,@path);
         # Handle multifields
-        if( exists $ref->{fields} && ref $ref->{fields} eq 'HASH') {
+        if( exists $ref->{fields} && is_hashref($ref->{fields}) ) {
             foreach my $k (sort keys %{ $ref->{fields} } ) {
                 _add_fields($f,@path,$k);
             }
@@ -1018,8 +1018,8 @@ sub _find_fields {
         debug({stderr=>1,color=>'red'},
             sprintf "_find_fields(): Invalid property at: %s ref info: %s",
                 join('.', @path),
-                join(',', ref $ref eq 'HASH' ? sort keys %{$ref} :
-                          ref $ref           ? ref $ref : 'unknown ref'
+                join(',', is_hashref($ref) ? sort keys %{$ref} :
+                          ref $ref         ? ref $ref : 'unknown ref'
                 ),
         );
     }
@@ -1082,7 +1082,7 @@ sub es_optimize_index {
 sub es_apply_index_settings {
     my($index,$settings) = @_;
 
-    if(ref $settings ne 'HASH') {
+    if(!is_hashref($settings)) {
         output({stderr=>1,color=>'red'}, 'usage is es_apply_index_settings($index,$settings_hashref)');
         return;
     }

diff --git a/lib/App/ElasticSearch/Utilities/Connection.pm b/lib/App/ElasticSearch/Utilities/Connection.pm
@@ -37,7 +37,7 @@ use JSON::MaybeXS;
 use LWP::UserAgent;
 use Module::Load;
 use Ref::Util qw(is_ref is_arrayref is_hashref);
-use Sub::Quote;
+use Types::Standard qw( Enum InstanceOf Int Str );
 use URI;
 use URI::QueryParam;
 
@@ -51,8 +51,8 @@ Hostname or ip to connect to, default 'B<localhost>'
 =cut
 has 'host' => (
     is      => 'ro',
-    isa     => quote_sub(q{ die "must specify a hostname or ip for host parameter" unless defined $_[0] and length $_[0] }),
-    default => quote_sub(q{'localhost'}),
+    isa     => Str,
+    default => sub { 'localhost' },
 );
 
 =attr port
@@ -62,8 +62,8 @@ Port to connect the HTTP transport for the ElasticSearch cluster, default is B<9
 =cut
 has 'port' => (
     is      => 'ro',
-    isa     => quote_sub(q{ die "must specify a port number" unless defined $_[0] and $_[0] =~ /^\d+$/ }),
-    default => quote_sub(q{9200}),
+    isa     => Int,
+    default => sub { 9200 },
 );
 
 =attr proto
@@ -78,8 +78,8 @@ basic authentication.
 
 has 'proto' => (
     is      => 'ro',
-    isa     => quote_sub(q{ die "must specify a protocol either http or https" unless defined $_[0] and $_[0] =~ /^http(s)?/}),
-    default => quote_sub(q{'http'}),
+    isa     => Enum[qw(http https)],
+    default => sub { 'http' },
 );
 
 =attr timeout
@@ -90,8 +90,8 @@ Connection and Read Timeout for the HTTP connection, defaults to B<10> seconds.
 
 has 'timeout' => (
     is      => 'ro',
-    isa     => quote_sub(q{ die "must specify a timeout in seconds" unless defined $_[0] and $_[0] =~ /^\d+$/ }),
-    default => quote_sub(q{10}),
+    isa     => Int,
+    default => sub { 10 },
 );
 
 =attr ua
@@ -102,7 +102,7 @@ Lazy built B<LWP::UserAgent> to access LWP::UserAgent directly.
 
 has 'ua' => (
     is  => 'lazy',
-    isa => quote_sub(q{die "UA setup failed." unless ref($_[0]) =~ /^LWP::UserAgent/}),
+    isa => InstanceOf["LWP::UserAgent"],
 );
 
 
@@ -113,7 +113,7 @@ has 'ua' => (
     sub LWP::UserAgent::get_basic_credentials {
         my ($self,$realm,$url) = @_;
         my $uri = URI->new( $url );
-        load App::ElasticSearch::Utilities => 'es_basic_auth';
+        load "App::ElasticSearch::Utilities" => 'es_basic_auth';
         return es_basic_auth( $uri->host );
     }
 }

diff --git a/lib/App/ElasticSearch/Utilities/HTTPRequest.pm b/lib/App/ElasticSearch/Utilities/HTTPRequest.pm
@@ -29,7 +29,7 @@ sub new {
 }
 
 sub content {
-	my ($self,$body) = @_;
+    my ($self,$body) = @_;
 
     if( is_arrayref($body) ) {
         # Bulk does this

diff --git a/lib/App/ElasticSearch/Utilities/Query.pm b/lib/App/ElasticSearch/Utilities/Query.pm
@@ -7,17 +7,13 @@ use warnings;
 use CLI::Helpers qw(:output);
 use Clone qw(clone);
 use Moo;
+use Ref::Util qw(is_arrayref is_hashref);
+use Types::Standard qw(ArrayRef HashRef Int Maybe Str);
+use Types::ElasticSearch qw(TimeConstant is_TimeConstant);
 use namespace::autoclean;
-use Sub::Quote;
 
-my %VALID = (
-    array_ref     => quote_sub(q{die "must be an array reference" if defined $_[0] and ref $_[0] ne 'ARRAY'}),
-    hash_ref      => quote_sub(q{die "must be a hash reference" if defined $_[0] and ref $_[0] ne 'HASH'}),
-    time_constant => quote_sub(q{die "must be time constant: https://www.elastic.co/guide/en/elasticsearch/reference/master/common-options.html#time-units" if defined $_[0] && $_[0] !~ /^\d+(y|M|w|d|h|m|s|ms)$/ }),
-    integer       => quote_sub(q{die "must be 0+ and integer" if defined $_[0] and $_[0] !~ /^\d+$/ }),
-);
 my %TO = (
-    array_ref => quote_sub(q{defined $_[0] && ref $_[0] eq 'ARRAY' ? $_[0] : defined $_[0] ? [ $_[0] ] : $_[0]}),
+    array_ref => sub { defined $_[0] && ref($_[0]) eq 'ARRAY' ? $_[0] : defined $_[0] ? [ $_[0] ] : $_[0] },
 );
 
 =attr query_stash
@@ -28,10 +24,10 @@ Hash reference containing replaceable query elements.  See L<stash>.
 
 has query_stash => (
     is       => 'rw',
+    isa      => HashRef,
     lazy     => 1,
     init_arg => undef,
     default  => sub {{}},
-    isa      => $VALID{hash_ref},
 );
 
 =attr must
@@ -54,13 +50,24 @@ Can be set using set_should and is a valid init_arg.
 The filter section of a bool query as an array reference.  See: L<add_bool>
 Can be set using set_filter and is a valid init_arg.
 
+=nested
+
+The nested query, this shortcircuits the rest of the query due to restrictions
+on the nested queries.
+
+=nested_path
+
+The path by being nested, only used in nested queries.
+
 =cut
 
 my %QUERY = (
-    must     => { default => sub {undef}, isa => $VALID{array_ref}, coerce => $TO{array_ref}, init_arg => 'must'     },
-    must_not => { default => sub {undef}, isa => $VALID{array_ref}, coerce => $TO{array_ref}, init_arg => 'must_not' },
-    should   => { default => sub {undef}, isa => $VALID{array_ref}, coerce => $TO{array_ref}, init_arg => 'should'   },
-    filter   => { default => sub {undef}, isa => $VALID{array_ref}, coerce => $TO{array_ref}, init_arg => 'filter'   },
+    must        => { default => sub {undef}, isa => Maybe[ArrayRef], coerce => $TO{array_ref}, init_arg => 'must'     },
+    must_not    => { default => sub {undef}, isa => Maybe[ArrayRef], coerce => $TO{array_ref}, init_arg => 'must_not' },
+    should      => { default => sub {undef}, isa => Maybe[ArrayRef], coerce => $TO{array_ref}, init_arg => 'should'   },
+    filter      => { default => sub {undef}, isa => Maybe[ArrayRef], coerce => $TO{array_ref}, init_arg => 'filter'   },
+    nested      => { default => sub {undef}, isa => Maybe[HashRef], init_arg => 'nested' },
+    nested_path => { default => sub {undef}, isa => Maybe[Str],     init_arg => 'nested_path' },
 );
 
 =attr from
@@ -95,11 +102,11 @@ Aliased as B<aggs>.
 =cut
 
 my %REQUEST_BODY = (
-    from         => { default => sub {undef}, isa => $VALID{integer} },
-    size         => { default => sub {50},    isa => $VALID{integer} },
-    fields       => { default => sub {undef}, isa => $VALID{array_ref}, coerce => $TO{array_ref} },
-    sort         => { default => sub {undef}, isa => $VALID{array_ref}, coerce => $TO{array_ref} },
-    aggregations => { default => sub {undef}, isa => $VALID{hash_ref} },
+    from         => { isa => Maybe[Int] },
+    size         => { default => sub {50},    isa => Int },
+    fields       => { isa => Maybe[ArrayRef], coerce => $TO{array_ref} },
+    sort         => { isa => Maybe[ArrayRef], coerce => $TO{array_ref} },
+    aggregations => { isa => Maybe[HashRef] },
 );
 
 =attr scroll
@@ -125,35 +132,35 @@ Can be set with B<set_terminateafter>.  Cannot be an init_arg.
 =cut
 
 my %PARAMS = (
-    scroll          => { default => sub {undef}, isa => $VALID{time_constant} },
-    timeout         => { default => sub {undef}, isa => $VALID{time_constant} },
-    terminate_after => { default => sub {undef}, isa => $VALID{integer} },
+    scroll          => { isa => Maybe[TimeConstant] },
+    timeout         => { isa => TimeConstant },
+    terminate_after => { isa => Int },
 );
 
 # Dynamically build our attributes
 foreach my $attr (keys %QUERY) {
     has $attr => (
-        is => 'rw',
-        lazy => 1,
-        writer => "set_$attr",
+        is       => 'rw',
+        lazy     => 1,
+        writer   => "set_$attr",
         init_arg => undef,
         %{ $QUERY{$attr} },
     );
 }
 foreach my $attr (keys %REQUEST_BODY) {
     has $attr => (
-        is => 'rw',
-        lazy => 1,
-        writer => "set_$attr",
+        is       => 'rw',
+        lazy     => 1,
+        writer   => "set_$attr",
         init_arg => undef,
         %{ $REQUEST_BODY{$attr} },
     );
 }
 foreach my $attr (keys %PARAMS) {
     has $attr => (
-        is => 'rw',
-        lazy => 1,
-        writer => "set_$attr",
+        is       => 'rw',
+        lazy     => 1,
+        writer   => "set_$attr",
         init_arg => undef,
         %{ $PARAMS{$attr} },
     );
@@ -221,26 +228,39 @@ the query will not be represented in the hash it returns.
 sub query {
     my $self = shift;
 
-    my %bool = ();
-    foreach my $k (keys %QUERY) {
-        no strict 'refs';
-        $bool{$k} = [];
-        my $v;
-        eval {
-            debug({color=>'yellow'}, "query() - retrieving section '$k'");
-            $v = $self->$k();
-            debug_var({color=>'cyan'},$v) if defined $v and ref $v;
-            1;
-        } or do {
-            debug({color=>'red'}, "query() - Failed to retrieve '$k'");
+    my $qref;
+    if( $self->nested ) {
+        $qref = {
+            nested => {
+                path  => $self->nested_path,
+                query => $self->nested,
+            }
         };
-        $bool{$k} = clone $v if defined $v;
-        if($self->stash($k)) {
-            push @{ $bool{$k} }, $self->stash($k);
+    }
+    else {
+        my %bool = ();
+        foreach my $k (keys %QUERY) {
+            next if $k =~ /^nested/;
+            no strict 'refs';
+            $bool{$k} = [];
+            my $v;
+            eval {
+                debug({color=>'yellow'}, "query() - retrieving section '$k'");
+                $v = $self->$k();
+                debug_var({color=>'cyan'},$v) if defined $v and ref $v;
+                1;
+            } or do {
+                debug({color=>'red'}, "query() - Failed to retrieve '$k'");
+            };
+            $bool{$k} = clone $v if defined $v;
+            if($self->stash($k)) {
+                push @{ $bool{$k} }, $self->stash($k);
+            }
+            delete $bool{$k} if exists $bool{$k} and not @{ $bool{$k} };
         }
-        delete $bool{$k} if exists $bool{$k} and not @{ $bool{$k} };
+        $qref = { bool => \%bool };
     }
-    return { bool => \%bool };
+    return $qref;
 }
 
 =method add_aggregations( name => { ...  } )
@@ -346,11 +366,9 @@ sub set_scan_scroll {
     my ($self,$ctxt_life) = @_;
 
     # Validate Context Lifetime
-    eval {
-        $VALID{time_constant}->($ctxt_life);
-    } or do {
+    if( !is_TimeConstant( $ctxt_life) ) {
         undef($ctxt_life);
-    };
+    }
     $ctxt_life ||= '1m';
 
     $self->set_sort( [qw(_doc)] );