Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added basic test framework, so functionality can be simple check (reg…

…ression etc)

Also added a new feature, multiple reports for each rule.
IGNORE in ACTION isn't working yet in this reimplementation
  • Loading branch information...
commit d6844ae007309b17cb94245f131df58a4256ae81 1 parent 4a6fb9d
Michael Knox authored
View
64 logparse.pl
@@ -849,21 +849,22 @@ sub report {
print "\n\nSummaries:\n";
for my $rule (sort keys %{ $$cfghashref{RULE} }) {
next if $rule =~ /nomatch/;
- my %ruleresults = summariseresults(\%{ $$cfghashref{RULE}{$rule} }, \%{ $$reshashref{$rule} });
- logmsg (5, 2, "\%ruleresults ... ".Dumper(%ruleresults));
+
if (exists ($$cfghashref{RULE}{$rule}{reports} ) ) {
- for my $rptid ($#{ $$cfghashref{RULE}{$rule}{reports} } ) {
- logmsg (4, 1, "Rule: $rule\[$rptid\]");
+ for my $rptid (0 .. $#{ $$cfghashref{RULE}{$rule}{reports} } ) {
+ my %ruleresults = summariseresults(\%{ $$cfghashref{RULE}{$rule} }, \%{ $$reshashref{$rule} });
+ logmsg (4, 1, "Rule[rptid]: $rule\[$rptid\]");
+ logmsg (5, 2, "\%ruleresults ... ".Dumper(%ruleresults));
if (exists ($$cfghashref{RULE}{$rule}{reports}[$rptid]{title} ) ) {
print "$$cfghashref{RULE}{$rule}{reports}[$rptid]{title}\n";
} else {
logmsg (4, 2, "No title");
}
- for my $key (keys %ruleresults) {
+ for my $key (keys %{$ruleresults{$rptid} }) {
logmsg (5, 3, "key:$key");
if (exists ($$cfghashref{RULE}{$rule}{reports}[$rptid]{line})) {
- print "\t".rptline(\% {$$cfghashref{RULE}{$rule}{reports}[$rptid] }, \%{$ruleresults{$key} }, $rule, $rptid, $key)."\n";
+ print "\t".rptline(\% {$$cfghashref{RULE}{$rule}{reports}[$rptid] }, \%{$ruleresults{$rptid} }, $rule, $rptid, $key)."\n";
} else {
print "\t$$reshashref{$rule}{$key}: $key\n";
}
@@ -875,24 +876,49 @@ sub report {
} #sub
sub summariseresults {
- my $cfghashref = shift; # passed $cfghash{RULE}{$rule}{actions}
+ my $cfghashref = shift; # passed $cfghash{RULE}{$rule}
my $reshashref = shift; # passed $reshashref{$rule}
#my $rule = shift;
# loop through reshash for a given rule and combine the results of all the actions
- # sum and count totals are summed together
- # AVG is avg'd against the running total
- # won't make sense if there are avg and sum/count actions, but this wouldn't make a lot of sense anyway
# returns a summarised hash
my %results;
for my $actionid (keys( %$reshashref ) ) {
for my $key (keys %{ $$reshashref{$actionid} } ) {
- $results{$key}{count} += $$reshashref{$actionid}{$key}{count};
- $results{$key}{total} += $$reshashref{$actionid}{$key}{total};
- $results{$key}{avg} = $results{$key}{total} / $results{$key}{count};
+ (my @values) = split (/:/, $key);
+ logmsg (9, 5, "values from key($key) ... @values");
+ for my $rptid (0 .. $#{ $$cfghashref{reports} }) {
+ (my @targetfieldids) = $$cfghashref{reports}[$rptid]{line} =~ /(\d+?)/g;
+ logmsg (9, 5, "targetfields (from $$cfghashref{reports}[$rptid]{line})... ".Dumper(@targetfieldids));
+
+ my $targetkeymatch;# my $targetid;
+ # create key using only fieldids required for rptline
+ # field id's are relative to the fields collected by the action cmds
+ for my $resfieldid (0 .. $#values) {# loop through the fields in the key from reshash (generated by action)
+ my $fieldid = $resfieldid+1;
+
+ logmsg (9, 6, "Checking for $fieldid in \@targetfieldids(@targetfieldids), \@values[$resfieldid] = $values[$resfieldid]");
+ if (grep(/^$fieldid$/, @targetfieldids ) ) {
+ $targetkeymatch .= ":$values[$resfieldid]";
+ logmsg(9, 7, "fieldid: $fieldid occured in @targetfieldids, adding $values[$resfieldid] to \$targetkeymatch");
+ } else {
+ $targetkeymatch .= ":.*?";
+ logmsg(9, 7, "fieldid: $fieldid doesn't in @targetfieldids, adding wildcard to \$targetkeymatch");
+ }
+ }
+ $targetkeymatch =~ s/^://;
+
+ logmsg (9, 5, "targetkey is $targetkeymatch");
+ if ($key =~ /$targetkeymatch/) {
+ $results{$rptid}{$targetkeymatch}{count} += $$reshashref{$actionid}{$key}{count};
+ $results{$rptid}{$targetkeymatch}{total} += $$reshashref{$actionid}{$key}{total};
+ $results{$rptid}{$targetkeymatch}{avg} = $$reshashref{$actionid}{$key}{total} / $$reshashref{$actionid}{$key}{count};
+ }
+ } # for $rptid in reports
} # for rule/actionid/key
} # for rule/actionid
+
return %results;
}
@@ -918,16 +944,16 @@ sub rptline {
for ($$cfghashref{cmd}) {
if (/count/i) {
- $line =~ s/\{x\}/$$reshashref{count}/;
- logmsg(9, 3, "subsituting {x} with $$reshashref{count}");
+ $line =~ s/\{x\}/$$reshashref{$key}{count}/;
+ logmsg(9, 3, "subsituting {x} with $$reshashref{$key}{count} (count)");
} elsif (/SUM/i) {
- $line =~ s/\{x\}/$$reshashref{total}/;
- logmsg(9, 3, "subsituting {x} with $$reshashref{total}");
+ $line =~ s/\{x\}/$$reshashref{$key}{total}/;
+ logmsg(9, 3, "subsituting {x} with $$reshashref{$key}{total} (total)");
} elsif (/AVG/i) {
- my $avg = $$reshashref{total} / $$reshashref{count};
+ my $avg = $$reshashref{$key}{total} / $$reshashref{$key}{count};
$avg = sprintf ("%.3f", $avg);
$line =~ s/\{x\}/$avg/;
- logmsg(9, 3, "subsituting {x} with $avg");
+ logmsg(9, 3, "subsituting {x} with $avg (avg)");
} else {
logmsg (1, 0, "WARNING: Unrecognized cmd ($$cfghashref{cmd}) for report #$rptid in rule ($rule)");
}
View
2  testcases/1/Description
@@ -0,0 +1,2 @@
+Basic test case for multiple reports in a rule & counts.
+Also IGNORE in an ACTION
View
21 testcases/1/expected.output
@@ -0,0 +1,21 @@
+
+
+No match for lines:
+ Oct 2 15:00:01 hosta CRON[3836]: pam_unix(cron:session): session opened for user root by (uid=0)
+ Oct 2 15:00:02 hosta CRON[3836]: pam_unix(cron:session): session closed for user root
+
+
+Summaries:
+Sudo usage by cmd/user/host
+ userb ran /usr/bin/tail /var/log/auth.log as root on hosta: 2 times
+ usera ran /bin/grep ssh /var/log/syslog as root on hosta: 2 times
+ userb ran /bin/grep -i ssh /var/log/apache2 /var/log/apparmor /var/log/apt /var/log/aptitude /var/log/aptitude.1.gz /var/log/auth.log /var/log/auth.log.0 /var/log/auth.log.1.gz /var/log/boot /var/log/btmp /var/log/btmp.1 /var/log/ConsoleKit /var/log/daemon.log /var/log/dbconfig-common /var/log/debug /var/log/dist-upgrade /var/log/dmesg /var/log/dmesg.0 /var/log/dmesg.1.gz /var/log/dmesg.2.gz /var/log/dmesg.3.gz /var/log/dmesg.4.gz /var/log/dpkg.log /var/log/dpkg.log.1 /var/log/dpkg.log.2.gz /var/log/exim4 /var/log/faillog /var/log/fsck /var/log/installer /var/log/kern.log /var/log/landscape /var/log/lastlog /var/log/lpr.log /var/log/mail.err /var/log/mail.info /var/log/mail.log /var/log/mail.warn /var/log/messages /var/log/mysql /var/log/mysql.err /var/log/mysql.log /var/log/mysql.log.1.gz /var/log/news /var/log/pycentral.log /var/log/request-tracker3.6 /var/log/syslog /var/log/syslog as root on hosta: 1 times
+ userb ran /bin/grep -i ssh /var/log/messages as root on hosta: 1 times
+ usera ran /bin/grep ssh /var/log/messages as root on hosta: 1 times
+ usera ran /bin/chown usera auth.log as root on hosta: 1 times
+ userb ran /usr/bin/head /var/log/auth.log as root on hosta: 1 times
+
+Sudo usage by user/host
+ usera ran 4 sudo cmds on hosta
+ userb ran 5 sudo cmds on hosta
+
View
44 testcases/1/logparse.conf
@@ -0,0 +1,44 @@
+#########
+FORMAT syslog {
+ # FIELDS <field count> <delimiter>
+ # FIELDS <field count> <delimiter> [<parent field>]
+ # FIELD <field id #> <label>
+ # FIELD <parent field id#>-<child field id#> <label>
+
+ FIELDS 6 "\s+"
+ FIELD 1 MONTH
+ FIELD 2 DAY
+ FIELD 3 TIME
+ FIELD 4 HOST
+ FIELD 5 APP
+ FIELD 6 FULLMSG
+ FIELDS 6-2 /]\s+/
+ FIELD 6-1 FACILITY
+ FIELD 6-2 MSG default
+
+ LASTLINEINDEX HOST
+ # LASTLINE <field label> # field to be used as index for "last line", also need to track rule matches for last line
+
+ # default format to be used for log file analysis
+ DEFAULT
+}
+
+RULE {
+ MSG /^message repeated (.*) times/ OR /^message repeated$/
+
+ ACTION MSG /^message repeated (.*) times/ {1} sum {1} append LASTLINE
+ # Apply {1} as {x} in the previously applied rule for HOST
+ ACTION MSG /^message repeated$/ count append LASTLINE
+}
+
+RULE sudo {
+ APP sudo
+ MSG /.* : .* ; PWD=/
+ ACTION MSG /(.*): TTY=(.*) ; PWD=(.*); USER=(.*); COMMAND=(.*)/ {HOST, 1, 4, 5}
+ ACTION MSG /(.*): (command continued) / IGNORE
+ REPORT "Sudo usage by cmd/user/host" "{2} ran {4} as {3} on {1}: {x} times" count
+ REPORT "Sudo usage by user/host" "{2} ran {x} sudo cmds on {1}" count
+ # by default {x} is total
+}
+#######################
+#2
View
12 testcases/1/test.log
@@ -0,0 +1,12 @@
+Oct 2 15:00:01 hosta CRON[3836]: pam_unix(cron:session): session opened for user root by (uid=0)
+Oct 2 15:00:02 hosta CRON[3836]: pam_unix(cron:session): session closed for user root
+Oct 2 15:01:19 hosta sudo: usera : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/bin/grep ssh /var/log/syslog
+Oct 2 15:01:22 hosta sudo: usera : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/bin/grep ssh /var/log/syslog
+Oct 2 15:01:25 hosta sudo: usera : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/bin/grep ssh /var/log/messages
+Oct 2 15:01:31 hosta sudo: userb : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/bin/grep -i ssh /var/log/messages
+Oct 2 15:01:35 hosta sudo: userb : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/bin/grep -i ssh /var/log/apache2 /var/log/apparmor /var/log/apt /var/log/aptitude /var/log/aptitude.1.gz /var/log/auth.log /var/log/auth.log.0 /var/log/auth.log.1.gz /var/log/boot /var/log/btmp /var/log/btmp.1 /var/log/ConsoleKit /var/log/daemon.log /var/log/dbconfig-common /var/log/debug /var/log/dist-upgrade /var/log/dmesg /var/log/dmesg.0 /var/log/dmesg.1.gz /var/log/dmesg.2.gz /var/log/dmesg.3.gz /var/log/dmesg.4.gz /var/log/dpkg.log /var/log/dpkg.log.1 /var/log/dpkg.log.2.gz /var/log/exim4 /var/log/faillog /var/log/fsck /var/log/installer /var/log/kern.log /var/log/landscape /var/log/lastlog /var/log/lpr.log /var/log/mail.err /var/log/mail.info /var/log/mail.log /var/log/mail.warn /var/log/messages /var/log/mysql /var/log/mysql.err /var/log/mysql.log /var/log/mysql.log.1.gz /var/log/news /var/log/pycentral.log /var/log/request-tracker3.6 /var/log/syslog /var/log/syslog
+Oct 2 15:01:35 hosta sudo: userb : (command continued) /var/log/syslog.1.gz /var/log/syslog.2.gz /var/log/syslog.3.gz /var/log/syslog.4.gz /var/log/syslog.5.gz /var/log/syslog.6.gz /var/log/udev /var/log/user.log /var/log/wtmp /var/log/wtmp.1
+Oct 2 15:02:17 hosta sudo: userb : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/usr/bin/head /var/log/auth.log
+Oct 2 15:02:22 hosta sudo: usera : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/bin/chown usera auth.log
+Oct 2 15:02:55 hosta sudo: userb : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/usr/bin/tail /var/log/auth.log
+Oct 2 15:03:50 hosta sudo: userb : TTY=pts/0 ; PWD=/home/mike/src/LogParse ; USER=root ; COMMAND=/usr/bin/tail /var/log/auth.log
View
15 testcases/test.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+dir=`dirname $0`
+
+for testcase in `ls -dF1 $dir/* | grep '/$'`
+do
+ echo Testcase: $testcase
+ $1 -l $testcase/test.log -c $testcase/logparse.conf -d0 > tmp.output
+ diff -u tmp.output $testcase/expected.output
+ if test $? -eq 0 ; then
+ echo $testcase passed
+ else
+ echo $testcase failed
+ fi
+done
Please sign in to comment.
Something went wrong with that request. Please try again.