Permalink
Browse files

Add new `fromapache` and `fromxferlog` operation.

  • Loading branch information...
1 parent cdc23b5 commit 5d322a9ca5aaf6983990ea3f18906c3cce31906f @bokutin committed Feb 22, 2014
View
1 bin/recs-fromapache
View
1 bin/recs-fromxferlog
View
79 lib/App/RecordStream/Operation/fromapache.pm
@@ -0,0 +1,79 @@
+package App::RecordStream::Operation::fromapache;
+
+our $VERSION = "4.0.4";
+
+use strict;
+use warnings;
+
+use base qw(App::RecordStream::Operation);
+
+use App::RecordStream::Record;
+use App::RecordStream::OptionalRequire qw(Apache::Log::Parser);
+App::RecordStream::OptionalRequire::require_done();
+
+sub init {
+ my $this = shift;
+ my $args = shift;
+
+ my $fast = 0;
+ my $strict = 0;
+ my $verbose = 0;
+
+ my $spec = {
+ "fast" => \$fast,
+ "strict" => \$strict,
@tsibley
tsibley added a line comment Feb 22, 2014

You need to specify that fast and strict optionally take a value, otherwise they're just boolean flags and the eval of them below won't work as expected.

Can you add tests of this functionality (i.e. specifying a Perl data structure for fast and strict)?

@tsibley
tsibley added a line comment Feb 22, 2014

Using the option specs fast:s and strict:s means the value is optional. When no value is passed, the value will be set to the empty string. You'll need to convert it to a true value by leaving the flags undef in your declaration and then checking defined-ness.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ "verbose" => \$verbose,
+ };
+
+ $this->parse_options($args, $spec);
+
+ $this->{'PARSER'} = Apache::Log::Parser->new(
+ ( $fast ? ( fast => eval $fast ) : () ),
+ ( $strict ? ( strict => eval $strict ) : () ),
+ ( $verbose ? ( verbose => $verbose ) : () ),
+ );
+}
+
+sub accept_line {
+ my $this = shift;
+ my $line = shift;
+
+ my $parser = $this->{'PARSER'};
+
+ if (my $hash = $parser->parse($line)) {
+ my $record = App::RecordStream::Record->new($hash);
+ $this->push_record($record);
+ }
+
+ return 1;
+}
+
+sub usage {
+ my $this = shift;
+
+ my $options = [
+ [ 'fast', q{'fast' parser works relatively fast. It can process only 'common', 'combined' and custom styles with compatibility with 'common', and cannot work with backslash-quoted double-quotes in fields.} ],
+ [ 'strict', q{'strict' parser works relatively slow. It can process any style format logs, with specification about separator, and checker for perfection. It can also process backslash-quoted double-quotes properly.} ],
+ [ 'verbose', q{Verbose output.} ],
+ ];
+
+ my $args_string = $this->options_string($options);
+
+ return <<USAGE;
+Usage: recs-fromapache <args>
+ __FORMAT_TEXT__
+ Each line of input (or lines of <files>) is parse by Apache::Log::Parser to produce an output record.
+ __FORMAT_TEXT__
+
+Arguments:
+$args_string
+
+Examples:
+ Get records from typical apache log
+ recs-fromapache --fast < /var/log/httpd-access.log
+ A more detailed how to use
+ recs-fromapache --strict '[qw(combined common vhost_common)]' < /var/log/httpd-access.log
+USAGE
+}
+
+1;
View
50 lib/App/RecordStream/Operation/fromxferlog.pm
@@ -0,0 +1,50 @@
+package App::RecordStream::Operation::fromxferlog;
+
+our $VERSION = "4.0.4";
+
+use strict;
+use warnings;
+
+use base qw(App::RecordStream::Operation);
+
+use App::RecordStream::Record;
+use App::RecordStream::OptionalRequire qw(Net::FTPServer::XferLog);
+App::RecordStream::OptionalRequire::require_done();
+
+sub init {
+ my $this = shift;
+ my $args = shift;
+
+ my $spec = {};
+
+ $this->parse_options($args, $spec);
+}
+
+sub accept_line {
+ my $this = shift;
+ my $line = shift;
+
+ if (my $hash = Net::FTPServer::XferLog->parse_line($line)) {
+ my $record = App::RecordStream::Record->new($hash);
+ $this->push_record($record);
+ }
+
+ return 1;
+}
+
+sub usage {
+ my $this = shift;
+
+ return <<USAGE;
+Usage: recs-fromxferlog <args>
+ __FORMAT_TEXT__
+ Each line of input (or lines of <files>) is parse by Net::FTPServer::XferLog to produce an output record.
+ __FORMAT_TEXT__
+
+Examples:
+ Get records from typical xferlog
+ recs-fromxferlog < /var/log/xferlog
+USAGE
+}
+
+1;
View
27 tests/RecordStream/Operation/fromapache.t
@@ -0,0 +1,27 @@
+use Test::More qw(no_plan);
+use App::RecordStream::Test::Tester;
+
+BEGIN { use_ok( 'App::RecordStream::Operation::fromapache' ) };
+
+my $tester = App::RecordStream::Test::Tester->new('fromapache');
+
+my $input;
+my $output;
+
+# combined
+$input = <<INPUT;
+192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "-" "DoCoMo/2.0 P03B(c500;TB;W24H16)"
+INPUT
+$output = <<OUTPUT;
+{"request":"GET /x/i.cgi/net/0000/ HTTP/1.1","bytes":"9891","proto":"HTTP/1.1","timezone":"+0900","status":"200","time":"10:59:59","date":"07/Feb/2011","rhost":"192.168.0.1","path":"/x/i.cgi/net/0000/","datetime":"07/Feb/2011:10:59:59 +0900","logname":"-","user":"-","agent":"DoCoMo/2.0 P03B(c500;TB;W24H16)","method":"GET","referer":"-"}
+OUTPUT
+$tester->test_input(['--fast'], $input, $output);
+
+# common
+$input = <<INPUT;
+192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891
+INPUT
+$output = <<OUTPUT;
+{"request":"GET /x/i.cgi/net/0000/ HTTP/1.1","bytes":"9891","proto":"HTTP/1.1","timezone":"+0900","status":"200","time":"10:59:59","date":"07/Feb/2011","rhost":"192.168.0.1","path":"/x/i.cgi/net/0000/","datetime":"07/Feb/2011:10:59:59 +0900","logname":"-","user":"-","method":"GET"}
+OUTPUT
+$tester->test_input(['--fast'], $input, $output);
View
23 tests/RecordStream/Operation/fromxferlog.t
@@ -0,0 +1,23 @@
+use Test::More qw(no_plan);
+use App::RecordStream::Test::Tester;
+
+BEGIN { use_ok( 'App::RecordStream::Operation::fromxferlog' ) };
+
+my $tester = App::RecordStream::Test::Tester->new('fromxferlog');
+
+my $input;
+my $output;
+
+$input = <<INPUT;
+Mon Oct 1 17:09:23 2001 0 127.0.0.1 2611 FILENAME a _ o r tmbranno ftp 0 * c
+Mon Oct 1 17:09:27 2001 0 127.0.0.1 22 NAMEFILE a _ o r tmbranno ftp 0 * c
+Mon Oct 1 17:09:27 2001 0 127.0.0.1 22 file with spaces in it.zip a _ o r tmbranno ftp 0 * c
+Mon Oct 1 17:09:31 2001 0 127.0.0.1 7276 p1774034_11i_zhs.zip a _ o r tmbranno ftp 0 * c
+INPUT
+$output = <<OUTPUT;
+{"file_size":"2611","remote_host":"127.0.0.1","month":"Oct","current_time":"17:09:23","special_action_flag":"_","day_name":"Mon","direction":"o","service_name":"ftp","day":"1","access_mode":"r","completion_status":"c","authenticated_user_id":"*","transfer_type":"a","username":"tmbranno","authentication_method":"0","transfer_time":"0","filename":"FILENAME","year":"2001"}
+{"file_size":"22","remote_host":"127.0.0.1","month":"Oct","current_time":"17:09:27","special_action_flag":"_","day_name":"Mon","direction":"o","service_name":"ftp","day":"1","access_mode":"r","completion_status":"c","authenticated_user_id":"*","transfer_type":"a","username":"tmbranno","authentication_method":"0","transfer_time":"0","filename":"NAMEFILE","year":"2001"}
+{"file_size":"22","remote_host":"127.0.0.1","month":"Oct","current_time":"17:09:27","special_action_flag":"_","day_name":"Mon","direction":"o","service_name":"ftp","day":"1","completion_status":"c","access_mode":"r","authenticated_user_id":"*","transfer_type":"a","authentication_method":"0","username":"tmbranno","transfer_time":"0","filename":"file with spaces in it.zip","year":"2001"}
+{"file_size":"7276","remote_host":"127.0.0.1","month":"Oct","current_time":"17:09:31","special_action_flag":"_","day_name":"Mon","direction":"o","service_name":"ftp","day":"1","access_mode":"r","completion_status":"c","authenticated_user_id":"*","transfer_type":"a","username":"tmbranno","authentication_method":"0","transfer_time":"0","filename":"p1774034_11i_zhs.zip","year":"2001"}
+OUTPUT
+$tester->test_input([], $input, $output);

0 comments on commit 5d322a9

Please sign in to comment.