Skip to content
Fetching contributors…
Cannot retrieve contributors at this time
executable file 549 lines (485 sloc) 18.4 KB
#!/usr/bin/perl -w
#
###########################################################################
#
# File: nf2csv
#
# Purpose: nf2csv parses iptables log messages and prints them on stdout
# in comma separated value format. This is most useful for
# generating data for the AfterGlow project
# (http://afterglow.sourceforge.net) to visualize iptables log
# data.
#
# nf2csv is part of the psad project (http://www.cipherdyne.org/psad)
#
# Author: Michael Rash (mbr@cipherdyne.org)
#
# Credits: (see the CREDITS file)
#
# Copyright (C) 2006 Michael Rash (mbr@cipherdyne.org)
#
# License (GNU Public License):
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA
#
###########################################################################
#
use Getopt::Long 'GetOptions';
use strict;
my $version = '2.1.8-pre2';
### regex to match an ip address
my $ip_re = qr|(?:[0-2]?\d{1,2}\.){3}[0-2]?\d{1,2}|;
### main packet data structure
my %pkt_NF_init = (
### data link layer
'src_mac' => '',
'dst_mac' => '',
'intf' => '', ### FIXME in and out interfaces?
### network layer
'src' => '',
'dst' => '',
'proto' => '',
'ip_id' => -1,
'ttl' => -1,
'tos' => '',
'ip_len' => -1,
'itype' => -1,
'icode' => -1,
'ip_opts' => '',
'icmp_seq' => -1,
'icmp_id' => -1,
'frag_bit' => 0,
### transport layer
'sp' => -1,
'dp' => -1,
'win' => -1,
'flags' => -1,
'tcp_seq' => -1,
'tcp_ack' => -1,
'tcp_opts' => '',
'udp_len' => -1,
### extra fields for internals (fwsnort sid matching,
### Netfilter logging prefixes and chains, etc.)
'fwsnort_sid' => 0,
'chain' => '',
'log_prefix' => '',
'syslog_host' => '',
'timestamp' => ''
);
my $csv_fields = '';
my $csv_print_uniq = 0;
my $csv_line_limit = 0;
my $csv_start_line = 0;
my $csv_end_line = 0;
my $csv_regex = 0;
my $csv_neg_regex = 0;
my $nf_log_file = '';
my $print_ver = 0;
my $debug = 0;
my $help = 0;
### make Getopts case sensitive
Getopt::Long::Configure('no_ignore_case');
&usage(1) unless (GetOptions(
'Messages-file=s' => \$nf_log_file, # Specify the path to file containing
'fields=s' => \$csv_fields, # Specify list of CSV fields.
'uniq-lines' => \$csv_print_uniq, # Only print unique lines in CSV
# output.
'max-lines=i' => \$csv_line_limit, # Limit the number of CSV output
# lines.
'start-line=i' => \$csv_start_line, # Starting line in CSV file.
'end-line=i' => \$csv_end_line, # Ending line in CSV file.
'regex=s' => \$csv_regex, # Require additional regex match.
'neg-regex=s' => \$csv_neg_regex, # Require additional negative regex
'debug' => \$debug, # Run in debug mode.
'Version' => \$print_ver, # Print the nf2csv version and exit.
'help' => \$help, # Display help.
));
&usage(0) if $help;
### Print the version number and exit if -V given on the command line.
if ($print_ver) {
print "[+] psad v$version by Michael Rash <mbr\@cipherdyne.org>\n";
exit 0;
}
### see what we should be searching for
my ($tokens_ar, $match_criteria_ar) = &csv_tokens();
$csv_regex = qr/$csv_regex/ if $csv_regex;
$csv_neg_regex = qr/$csv_neg_regex/ if $csv_neg_regex;
my %csv_uniq_lines = ();
if ($csv_start_line) {
die "[*] Cannot have start line > end line."
if $csv_start_line > $csv_end_line;
}
my $ctr = 0;
my $line_ctr = 0;
my $fh = *STDIN;
if ($nf_log_file) {
open MSGS, "< $nf_log_file" or die "[*] Could not open ",
"$nf_log_file: $!";
$fh = *MSGS;
}
MSG: while (<$fh>) {
my $pkt_str = $_;
$line_ctr++;
if ($csv_start_line) {
next MSG unless $line_ctr >= $csv_start_line;
}
if ($csv_end_line) {
last MSG if $line_ctr == $csv_end_line;
}
next MSG unless $pkt_str =~ /IN.*OUT/;
### init pkt hash
my %pkt = %pkt_NF_init;
my $rv = &parse_NF_pkt_str(\%pkt, $pkt_str);
next MSG unless $rv;
if ($csv_regex) {
next MSG unless $pkt{'raw'} =~ m|$csv_regex|;
}
if ($csv_neg_regex) {
next MSG unless $pkt{'raw'} !~ m|$csv_neg_regex|;
}
$pkt{'log_prefix'} =~ s/\W//g;
$pkt{'log_prefix'} =~ s/\s//g;
my @matched_fields = ();
for (my $i=0; $i <= $#$tokens_ar; $i++) {
my $tok = $tokens_ar->[$i];
if ($match_criteria_ar) {
my $match_hr = $match_criteria_ar->[$i];
if (defined $match_hr->{'num'}) {
unless ($pkt{$tok} =~ m|^\d+$|
and $pkt{$tok} == $match_hr->{'num'}) {
next MSG;
}
} elsif (defined $match_hr->{'gt'}) {
unless ($pkt{$tok} =~ m|^\d+$|
and $pkt{$tok} > $match_hr->{'gt'}) {
next MSG;
}
} elsif (defined $match_hr->{'lt'}) {
unless ($pkt{$tok} =~ m|^\d+$|
and $pkt{$tok} < $match_hr->{'lt'}) {
next MSG;
}
} elsif (defined $match_hr->{'str'}) {
unless ($pkt{$tok} eq $match_hr->{'str'}) {
next MSG;
}
} elsif (defined $match_hr->{'re'}) {
unless ($pkt{$tok} =~ m|$match_hr->{'re'}m|) {
next MSG;
}
} elsif (defined $match_hr->{'net'}) {
if ($pkt{$tok} =~ m|$ip_re|) {
unless (ipv4_in_network($match_hr->{'net'}, $pkt{$tok})) {
next MSG;
}
} else {
next MSG;
}
} elsif (defined $match_hr->{'ip'}) {
unless ($pkt{$tok} eq $match_hr->{'ip'}) {
next MSG;
}
}
push @matched_fields, $pkt{$tok};
} else {
push @matched_fields, $pkt{$tok};
}
}
next MSG unless @matched_fields;
my $str = '';
if ($csv_fields) {
$str .= "$_, " for @matched_fields;
} else {
$str .= "$_ " for @matched_fields;
}
$str =~ s/,\s*$//;
$str =~ s/\s*$//;
$ctr++;
if ($csv_print_uniq) {
$csv_uniq_lines{$str} = '';
} else {
print $str, "\n";
}
if ($csv_line_limit > 0) {
last if $ctr >= $csv_line_limit;
}
}
close $fh;
if ($csv_print_uniq) {
print "$_\n" for keys %csv_uniq_lines;
}
exit 0;
#============================= end main ===============================
sub csv_tokens() {
my @tokens = ();
my @match_criteria = ();
if ($csv_fields) {
my @tok_tmp = split /\s+/, $csv_fields;
for my $tok_str (@tok_tmp) {
my $token = $tok_str;
my $search = '';
if ($tok_str =~ m|(\w+):(\S+)|) {
$token = $1;
$search = $2;
}
$token = 'src' if $token eq 'SRC';
$token = 'dst' if $token eq 'DST';
$token = 'sp' if $token eq 'SPT';
$token = 'dp' if $token eq 'DPT';
$token = 'tos' if $token eq 'TOS';
$token = 'win' if $token eq 'WIN';
$token = 'itype' if $token eq 'TYPE';
$token = 'icode' if $token eq 'CODE';
$token = 'ttl' if $token eq 'TTL';
$token = 'ip_id' if $token eq 'ID';
$token = 'icmp_seq' if $token eq 'SEQ';
$token = 'proto' if $token eq 'PROTO';
$token = 'ip_len' if $token eq 'LEN';
$token = 'intf' if $token eq 'IN' or $token eq 'OUT';
unless (defined $pkt_NF_init{$token}) {
print "[*] $token is not a valid packet field; valid ",
"fields are:\n";
for my $key (sort keys %pkt_NF_init) {
print " $key\n";
}
die;
}
push @tokens, $token;
if ($search) {
my %search_hsh = ();
if ($search =~ m|^\d+$|) {
$search_hsh{'num'} = $search;
} elsif ($search =~ m|^>(\d+)$|) {
$search_hsh{'gt'} = $1;
die "[*] $token value must be >= 0"
unless $1 >= 0;
} elsif ($search =~ m|^<(\d+)$|) {
$search_hsh{'lt'} = $1;
die "[*] $token value must be >= 0"
unless $1 >= 0;
} elsif ($search =~ m|^/(.*?)/$|) {
$search_hsh{'re'} = qr|$1|;
} elsif ($search =~ m|^\"(.*?)\"$|) {
$search_hsh{'str'} = $1;
} elsif ($search =~ m|^$ip_re/$ip_re$|) {
$search_hsh{'net'} = $search;
} elsif ($search =~ m|^$ip_re/\d+$|) {
$search_hsh{'net'} = $search;
} elsif ($search =~ m|^$ip_re$|) {
$search_hsh{'ip'} = $search;
} else {
die "[*] Unrecognized value for $token";
}
push @match_criteria, \%search_hsh;
} else {
push @match_criteria, {};
}
}
} else {
@tokens = qw(
timestamp
src
dst
sp
dp
proto
flags
ip_len
intf
chain
log_prefix
);
}
return \@tokens, \@match_criteria;
}
sub parse_NF_pkt_str() {
my ($pkt_hr, $pkt_str) = @_;
print STDERR "\n", $pkt_str if $debug;
$pkt_hr->{'raw'} = $pkt_str;
if ($pkt_str =~ /.*kernel:\s+(.*?)\s*IN=/) {
$pkt_hr->{'log_prefix'} = $1;
}
### get the in/out interface and iptables chain (the code below
### allows the iptables log message to contain the PHYSDEV stuff):
### Feb 25 12:13:27 bridge kernel: INBOUND TCP: IN=br0 PHYSIN=eth0 OUT=br0
### PHYSOUT=eth1 SRC=63.147.183.21 DST=11.11.79.100 LEN=48 TOS=0x00
### PREC=0x00 TTL=113 ID=19664 DF PROTO=TCP SPT=4918 DPT=135 WINDOW=64240
### RES=0x00 SYN URGP=0
if ($pkt_str =~ /\sIN=(\S+).*\sOUT=\s/) {
$pkt_hr->{'intf'} = $1;
$pkt_hr->{'chain'} = 'INPUT';
} elsif ($pkt_str =~ /\sIN=(\S+).*\sOUT=\S/) {
$pkt_hr->{'intf'} = $1;
$pkt_hr->{'chain'} = 'FORWARD';
} elsif ($pkt_str =~ /\sIN=\s+.*\sOUT=(\S+)/) {
$pkt_hr->{'intf'} = $1;
$pkt_hr->{'chain'} = 'OUTPUT';
}
if ($pkt_str =~ /\sMAC=(\S+)/) {
my $mac_str = $1;
if ($mac_str =~ /^((?:\w{2}\:){6})((?:\w{2}\:){6})/) {
$pkt_hr->{'dst_mac'} = $1;
$pkt_hr->{'src_mac'} = $2;
}
}
if ($pkt_hr->{'src_mac'}) {
$pkt_hr->{'src_mac'} =~ s/:$//;
print STDERR "[+] src mac addr: $pkt_hr->{'src_mac'}\n" if $debug;
}
if ($pkt_hr->{'dst_mac'}) {
$pkt_hr->{'dst_mac'} =~ s/:$//;
print STDERR "[+] dst mac addr: $pkt_hr->{'dst_mac'}\n" if $debug;
}
unless ($pkt_hr->{'intf'} and $pkt_hr->{'chain'}) {
print STDERR "[-] err packet: could not determine ",
"interface and chain.\n" if $debug;
return 0;
}
### get the syslog logging host for this packet
if ($pkt_str =~ /^\s*((?:\S+\s+){2}\S+)\s+(\S+)\s+kernel:/) {
$pkt_hr->{'timestamp'} = $1;
$pkt_hr->{'syslog_host'} = $2;
} else {
$pkt_hr->{'timestamp'} = localtime();
$pkt_hr->{'syslog_host'} = 'unknown';
}
### try to extract a snort sid (generated by fwsnort) from
### the packet
if ($pkt_str =~ /SID(\d+)/) {
$pkt_hr->{'fwsnort_sid'} = $1;
}
### get IP options if --log-ip-options is used
### (they appear before the PROTO= field).
if ($pkt_str =~ /OPT\s+\((\S+)\)\s+PROTO=/) {
$pkt_hr->{'ip_opts'} = $1;
}
### May 18 22:21:26 orthanc kernel: DROP IN=eth2 OUT=
### MAC=00:60:1d:23:d0:01:00:60:1d:23:d3:0e:08:00 SRC=192.168.20.25
### DST=192.168.20.1 LEN=60 TOS=0x10 PREC=0x00 TTL=64 ID=47300 DF
### PROTO=TCP SPT=34111 DPT=6345 WINDOW=5840 RES=0x00 SYN URGP=0
if ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+)\s+TOS=(\S+)
\s*.*\s+TTL=(\d+)\s+ID=(\d+)\s*.*\s+PROTO=TCP\s+
SPT=(\d+)\s+DPT=(\d+)\s.*\s*WINDOW=(\d+)\s+
(.*)\s+URGP=/x) {
($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
$pkt_hr->{'tos'}, $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'},
$pkt_hr->{'sp'}, $pkt_hr->{'dp'}, $pkt_hr->{'win'},
$pkt_hr->{'flags'})
= ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10);
### the reserve bits are not reported by ulogd, but normal
### iptables syslog messages contain them.
$pkt_hr->{'flags'} =~ s/\s*RES=\S+\s*//;
$pkt_hr->{'proto'} = 'tcp';
### default to NULL
$pkt_hr->{'flags'} = 'NULL' unless $pkt_hr->{'flags'};
unless ($pkt_hr->{'flags'} !~ /WIN/ &&
$pkt_hr->{'flags'} =~ /ACK/ ||
$pkt_hr->{'flags'} =~ /SYN/ ||
$pkt_hr->{'flags'} =~ /RST/ ||
$pkt_hr->{'flags'} =~ /URG/ ||
$pkt_hr->{'flags'} =~ /PSH/ ||
$pkt_hr->{'flags'} =~ /FIN/ ||
$pkt_hr->{'flags'} eq 'NULL') {
print STDERR "[-] err packet: bad tcp flags.\n" if $debug;
return 0;
}
$pkt_hr->{'frag_bit'} = 1 if $pkt_str =~ /\sDF\s+PROTO/;
### don't pickup IP options if --log-ip-options is used
### (they appear before the PROTO= field).
if ($pkt_str =~ /URGP=\S+\s+OPT\s+\((\S+)\)/) {
$pkt_hr->{'tcp_opts'} = $1;
}
### make sure we have a "reasonable" packet (note that nmap
### can scan port 0 and iptables can report this fact)
unless ($pkt_hr->{'ip_len'} >= 0 and $pkt_hr->{'tos'}
and $pkt_hr->{'ttl'} >= 0 and $pkt_hr->{'ip_id'} >= 0
and $pkt_hr->{'proto'} and $pkt_hr->{'sp'} >= 0
and $pkt_hr->{'dp'} >= 0 and $pkt_hr->{'win'} >= 0
and $pkt_hr->{'flags'}) {
return 0;
}
### May 18 22:21:26 orthanc kernel: DROP IN=eth2 OUT=
### MAC=00:60:1d:23:d0:01:00:60:1d:23:d3:0e:08:00
### SRC=192.168.20.25 DST=192.168.20.1 LEN=28 TOS=0x00 PREC=0x00
### TTL=40 ID=47523 PROTO=UDP SPT=57339 DPT=305 LEN=8
} elsif ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+)\s+TOS=(\S+)
\s.*TTL=(\d+)\s+ID=(\d+)\s*.*\s+PROTO=UDP\s+
SPT=(\d+)\s+DPT=(\d+)\s+LEN=(\d+)/x) {
($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
$pkt_hr->{'tos'}, $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'},
$pkt_hr->{'sp'}, $pkt_hr->{'dp'}, $pkt_hr->{'udp_len'})
= ($1,$2,$3,$4,$5,$6,$7,$8,$9);
$pkt_hr->{'proto'} = 'udp';
### make sure we have a "reasonable" packet (note that nmap
### can scan port 0 and iptables can report this fact)
unless ($pkt_hr->{'ip_len'} >= 0
and $pkt_hr->{'tos'} and $pkt_hr->{'ttl'} >= 0
and $pkt_hr->{'ip_id'} >= 0 and $pkt_hr->{'proto'}
and $pkt_hr->{'sp'} >= 0 and $pkt_hr->{'dp'} >= 0
and $pkt_hr->{'udp_len'} >= 0) {
return 0;
}
### Nov 27 15:45:51 orthanc kernel: DROP IN=eth1 OUT= MAC=00:a0:cc:e2:1f:f2:00:
### 20:78:10:70:e7:08:00 SRC=192.168.10.20 DST=192.168.10.1 LEN=84 TOS=0x00
### PREC=0x00 TTL=64 ID=0 DF PROTO=ICMP TYPE=8 CODE=0 ID=61055 SEQ=256
} elsif ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+).*
TTL=(\d+)\s+ID=(\d+).*PROTO=ICMP\s+TYPE=(\d+)\s+
CODE=(\d+)\s+ID=(\d+)\s+SEQ=(\d+)/x) {
($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
$pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'}, $pkt_hr->{'itype'},
$pkt_hr->{'icode'}, $pkt_hr->{'icmp_id'}, $pkt_hr->{'icmp_seq'})
= ($1,$2,$3,$4,$5,$6,$7,$8,$9);
$pkt_hr->{'proto'} = 'icmp';
$pkt_hr->{'sp'} = $pkt_hr->{'dp'} = 0;
unless ($pkt_hr->{'ip_len'} >= 0 and $pkt_hr->{'ttl'} >= 0
and $pkt_hr->{'proto'} and $pkt_hr->{'itype'} >= 0
and $pkt_hr->{'icode'} >= 0 and $pkt_hr->{'ip_id'} >= 0
and $pkt_hr->{'icmp_seq'} >= 0) {
return 0;
}
} else {
### Sometimes the iptables log entry gets messed up due to
### buffering issues so we write it to the error log.
print STDERR "[-] err packet: no regex match.\n" if $debug;
return 0;
}
return 1;
}
sub usage() {
my $exitcode = shift;
print <<_HELP_;
nf2csv
[+] Version: $version
[+] By Michael Rash (mbr\@cipherdyne.org, http://www.cipherdyne.org)
Usage: nf2csv [options]
Options:
-f, --fields <fields> - Restrict output to a list of
specfic fields.
-u, --uniq-lines - Only print unique lines
-m, --max-lines <num> - Specify the maximum number of
output lines to print.
-M, --messages-file <file> - Specify the path to the iptables
logfile (use in conjunction with
--Analyze-msgs).
-s, --start-line <line> - Starting line within iptables log
file.
-e, --end-line <line> - Ending line within iptables log
file.
-r, --regex <regex> - Require iptables log messages to
match an additional regex.
-n, --neg-regex <regex> - Require iptables log messages to
not match an additional regex.
-d, --debug - Run in debugging mode.
-V, --Version - Print the version and exit.
-h --help - Display usage on STDOUT and exit.
_HELP_
exit $exitcode;
}
Jump to Line
Something went wrong with that request. Please try again.