diff --git a/README.md b/README.md index b1c399f..28fe535 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,6 @@ This repository contains open source components and extensions for nProbe availa Here you can find: * Add-on and extensions -* Code example of applications that can interact with nProbe +* Code examples of applications that can interact with nProbe Enjoy! diff --git a/bgp/README b/bgp/README new file mode 100644 index 0000000..dfda85e --- /dev/null +++ b/bgp/README @@ -0,0 +1,13 @@ +BGP lister written in perl: your BGP-enabled router connects to it, and it injects in a local +nProbe instance all the routes (initially all received routes, later all the routing updates) +so that the probe can compute the AS path and put it into emitted flows. + +You can start this script from the nProbe BGP plugin part of nProbe Pro + + +NOTE that you need to configure into the source code your AS info + +Download: +- http://www.ris.ripe.net/source/libbgpdump-1.4.99.11.tar.gz +- http://data.ris.ripe.net/rrc10/2010.07/ + diff --git a/bgp/bgp_probe_client.pl b/bgp/bgp_probe_client.pl new file mode 100755 index 0000000..9d009b2 --- /dev/null +++ b/bgp/bgp_probe_client.pl @@ -0,0 +1,233 @@ +#!/usr/bin/perl + +#use strict; + +use Getopt::Std; + +use Net::BGP::Process; +use Net::BGP::Peer; +use Net::BGP::Refresh; + +use threads; +use Thread::Queue; + +############# + +# +# Configuration parameters +# + +# BGP +my $local_ip = '192.168.48.2'; +my $local_as = 65498; +my $remote_ip = '192.168.48.1'; +my $remote_as = 2597; + +# nProbe +my $nprobe_ip = '127.0.0.1'; +my $nprobe_port = 4096; + +############# + +my $max_queue_len = 32768; +my $debug = 0; +my $dump_file = ""; + +%options=(); +getopts("i:d:vh",\%options); + +help() if defined $options{h}; +$debug = 1 if defined $options{v}; +$dump_file = $options{d} if defined $options{d}; +($nprobe_ip,$nprobe_port) = split(/:/, $options{i}) if defined $options{i}; + +############ + +my $bgp = Net::BGP::Process->new(); +my $peer = Net::BGP::Peer->new( + Start => 1, + ThisID => $local_ip, + ThisAS => $local_as, + PeerID => $remote_ip, + PeerAS => $remote_as, + Passive => 1, + UpdateCallback => \&my_update_callback + ); + +my $refresh = Net::BGP::Refresh->new( + AFI => Net::BGP::AFI_IP4, + SAFI => Net::BGP::SAFI_BOTH, + ); + +my %as_paths = (); +my $num_updates : shared = 0; +my $num_dropped_updates : shared = 0; +my $cmdQueue = Thread::Queue->new; + +my $socket; + +############################ + +sub openSocket() { + $socket = IO::Socket::INET->new(PeerAddr => $nprobe_ip, + PeerPort => $nprobe_port, + Proto => "tcp"); + + if(defined $socket) { + print "New socket open...\n"; + } else { + print "Couldn't connect to $nprobe_host:$nprobe_port : $@\n"; + sleep 1; + } +} + +sub processCmds { + my $max_queue_len = 0; + my $OUT; + + if($dump_file ne "") { + # Dump mode + open OUT, '>', $dump_file or die $!; + } + + while (my $cmd = $cmdQueue->dequeue()) { + my $num = $cmdQueue->pending(); + if($num > $max_queue_len) { $max_queue_len = $num; } + if($debug) { print $cmd."\n"; } + + if($dump_file ne "") { + # Dump mode + print OUT $cmd; + } else { + # Socket mode + + if(not defined $socket) { + openSocket(); + } + + if(defined $socket) { + my $bytes_sent = $socket->send($cmd); + + if((not defined $bytes_sent) || ($bytes_sent == 0)) { + print "Socket was closed by remote peer\n"; + close($socket); + openSocket(); + } + } + } + } +} + +my $i=0; +my $num_threads = 1; + +for($i=0; $i<$num_threads; $i++) { + my $thr = threads->new(\&processCmds); + $thr->detach; # Now we officially don't care any more +} + +############################ + +$bgp->add_peer($peer); +$peer->refresh($refresh); +$peer->start(); +$bgp->event_loop(); + +sub my_update_callback +{ + my ($peer,$update) = @_; + my %h; + my $as_path; + + #print "Update from [$peer][$update]\n"; + + ################################ + + # Remove duplicates entries + my @path = uniq(split(/ /, $update->{_as_path})); + + #shift(@path); # Delete top element + + my $target_as = $path[$#path]; + #pop(@path); # Delete last element from array (i.e. remove target_as) + + if(!($target_as =~ m/^{/)) { + my $old_val = $as_paths{$target_as}; + + # Format: (number of elements)@(elem 1),(elem 2).... + $as_path = ($#path+1)."@".join(",", @path); + + #print $as_path."\n"; + if($old_val ne $as_path) { + $as_paths{$target_as} = $as_path; + } + + #if($debug) { print $as_path."\n"; } else { print "."; } + } else { + # Something bad happened + return; + } + + ######################## + + my @nlri = @{$update->nlri()}; + if($debug) { print "[$num_updates] [ "; } + foreach (@nlri) { + if($debug) { print $_." "; } + my $net = $_; + + if ($net =~ m/^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/) { + $cmd = "+".$net."=".$as_path."\n"; + if($debug) { print $cmd; } + if($cmdQueue->pending() < $max_queue_len) { + $cmdQueue->enqueue($cmd); + $num_updates++; + } else { + $num_dropped_updates++; + } + } + } + + ######################## + + my @withdrawn = @{$update->withdrawn()}; + if($debug) { print "[$num_updates] [ "; } + foreach (@withdrawn) { + if($debug) { print $_." "; } + my $net = $_; + + if ($net =~ m/^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/) { + $cmd = "-".$net."=".$as_path."\n"; + if($debug) { print $cmd; } + + if($cmdQueue->pending() < $max_queue_len) { + $cmdQueue->enqueue($cmd); + $num_updates++; + } else { + $num_dropped_updates++; + } + } + } + +} + +############ + +sub uniq { + my %seen = (); + my @r = (); + foreach my $a (@_) { + unless ($seen{$a}) { + push @r, $a; + $seen{$a} = 1; + } + } + return @r; +} + +############ + +sub help { + print "bgp_probe_client.pl [-i ] [-d ] [-v] [-h]\n"; + exit 0; +} diff --git a/splunk/README b/splunk/README new file mode 100644 index 0000000..00bb39f --- /dev/null +++ b/splunk/README @@ -0,0 +1 @@ +Example of integration of nProbe with Splunk. This package is a slightly outdated and it should be updated, but it gives a good understanding how nProbe can interact with Splunk \ No newline at end of file diff --git a/splunk/create_spl.sh b/splunk/create_spl.sh new file mode 100755 index 0000000..32ab293 --- /dev/null +++ b/splunk/create_spl.sh @@ -0,0 +1,5 @@ + #!/bin/bash + tar -cvzf nprobe.tar.gz nprobe/ + mv nprobe.tar.gz nprobe.spl + + # On Mac OS X, use gnutar rather than the default tar packaged with the OS. The default tar utility generates a series of warnings that can be problematic when packaging your app. \ No newline at end of file diff --git a/splunk/doc/Splunk App-QuickStart.zip b/splunk/doc/Splunk App-QuickStart.zip new file mode 100644 index 0000000..b5e8e84 Binary files /dev/null and b/splunk/doc/Splunk App-QuickStart.zip differ diff --git a/splunk/doc/nprobe_elasticsearch_kibana.graffle/data.plist b/splunk/doc/nprobe_elasticsearch_kibana.graffle/data.plist new file mode 100644 index 0000000..99958a5 --- /dev/null +++ b/splunk/doc/nprobe_elasticsearch_kibana.graffle/data.plist @@ -0,0 +1,1092 @@ + + + + + ActiveLayerIndex + 0 + ApplicationVersion + + com.omnigroup.OmniGrafflePro + 138.33.0.157554 + + AutoAdjust + + BackgroundGraphic + + Bounds + {{0, 0}, {559, 783}} + Class + SolidGraphic + ID + 2 + Style + + shadow + + Draws + NO + + stroke + + Draws + NO + + + + CanvasOrigin + {0, 0} + ColumnAlign + 1 + ColumnSpacing + 36 + CreationDate + 2014-02-04 21:52:01 +0000 + Creator + Filippo Fontanelli + DisplayScale + 1 0/72 in = 1.0000 in + GraphDocumentVersion + 8 + GraphicsList + + + Bounds + {{341, 182}, {46, 28}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 71 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs24 \cf0 Collector\ +Mode} + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{187, 611}, {35, 38}} + Class + ShapedGraphic + ID + 50 + ImageID + 8 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + + + Class + LineGraphic + Head + + ID + 53 + + ID + 51 + Points + + {246.44618, 517} + {246.30382, 599} + + Style + + stroke + + HeadArrow + FilledArrow + LineType + 1 + TailArrow + 0 + + + Tail + + ID + 59 + + + + Bounds + {{344, 616}, {52, 28}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 52 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs24 \cf0 Web \ +Interfaces} + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{168.75, 600}, {155, 60}} + Class + ShapedGraphic + ID + 53 + Shape + RoundRect + Style + + stroke + + Width + 2 + + + Text + + Align + 2 + RTFD + + BAtzdHJlYW10eXBlZIHoA4QBQISEhBJOU0F0dHJpYnV0 + ZWRTdHJpbmcAhIQITlNPYmplY3QAhZKEhIQITlNTdHJp + bmcBlIQBKwhLaWJhbmEgM4aEAmlJAQiShISEDE5TRGlj + dGlvbmFyeQCUhAFpA5KElpYHTlNDb2xvcoaShISEB05T + Q29sb3IAlIQBYwKEBGZmZmaDjo0NP4Pk4uI+g5WUlD4B + hpKElpYQTlNQYXJhZ3JhcGhTdHlsZYaShISEEE5TUGFy + YWdyYXBoU3R5bGUAlIQEQ0NAUwEAhISEB05TQXJyYXkA + lJkMkoSEhAlOU1RleHRUYWIAlIQCQ2YAHIaShKKhADiG + koSioQBUhpKEoqEAcIaShKKhAIGMAIaShKKhAIGoAIaS + hKKhAIHEAIaShKKhAIHgAIaShKKhAIH8AIaShKKhAIEY + AYaShKKhAIE0AYaShKKhAIFQAYaGAIaShJaWBk5TRm9u + dIaShISEBk5TRm9udB6UmSiEBVs0MGNdBgAAAB4AAAD/ + /lYAQQBHAFIAbwB1AG4AZABlAGQAQgBvAGwAZAAAAIQB + ZhKbAJsBmwCbAIaGhg== + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;\red122\green94\blue58;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qr + +\f0\b\fs36 \cf2 Kibana 3} + + + + Bounds + {{182, 469.5}, {30, 33}} + Class + ShapedGraphic + ID + 54 + ImageID + 7 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + + + Class + LineGraphic + Head + + ID + 57 + + ID + 55 + Points + + {246.5, 372} + {246.5, 406.5} + + Style + + stroke + + HeadArrow + 0 + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 65 + + + + Class + LineGraphic + Head + + ID + 59 + + ID + 56 + Points + + {246.5, 420.5} + {246.5, 455} + + Style + + stroke + + HeadArrow + FilledArrow + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 57 + + + + Bounds + {{210.5, 406.5}, {72, 14}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 57 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 Export Plugin} + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{337.5, 472}, {53, 28}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 58 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs24 \cf0 Search \ +& Storage} + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{169, 456}, {155, 60}} + Class + ShapedGraphic + ID + 59 + Shape + RoundRect + Style + + stroke + + Width + 2 + + + Text + + Align + 2 + RTFD + + BAtzdHJlYW10eXBlZIHoA4QBQISEhBJOU0F0dHJpYnV0 + ZWRTdHJpbmcAhIQITlNPYmplY3QAhZKEhIQITlNTdHJp + bmcBlIQBKwxFbGFzdGljc2VhcmOGhAJpSQEMkoSEhAxO + U0RpY3Rpb25hcnkAlIQBaQSShJaWEE5TUGFyYWdyYXBo + U3R5bGWGkoSEhBBOU1BhcmFncmFwaFN0eWxlAJSEBEND + QFMBAISEhAdOU0FycmF5AJSZDJKEhIQJTlNUZXh0VGFi + AJSEAkNmAByGkoSfngA4hpKEn54AVIaShJ+eAHCGkoSf + ngCBjACGkoSfngCBqACGkoSfngCBxACGkoSfngCB4ACG + koSfngCB/ACGkoSfngCBGAGGkoSfngCBNAGGkoSfngCB + UAGGhgCGkoSWlg5OU09yaWdpbmFsRm9udIaShISEBk5T + Rm9udB6UmSiEBVs0MGNdBgAAAB4AAAD//lYAQQBHAFIA + bwB1AG4AZABlAGQAQgBvAGwAZAAAAIQBZhKEAWMAogGi + AKIAhpKElpYGTlNGb250hpKskoSWlgdOU0NvbG9yhpKE + hIQHTlNDb2xvcgCUogKEBGZmZmaD7OrqPoO7ujo/g5eW + lj4BhoaG + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;\red100\green176\blue59;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qr + +\f0\b\fs36 \cf2 Elasticsearc} + + + + Bounds + {{192, 322}, {30, 38}} + Class + ShapedGraphic + ID + 60 + ImageID + 5 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + + + Bounds + {{344, 334}, {40, 14}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 61 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs24 \cf0 Indexer} + VerticalPad + 0 + + Wrap + NO + + + Class + LineGraphic + Head + + ID + 64 + + ID + 62 + Points + + {246.5, 227} + {246.5, 261.5} + + Style + + stroke + + HeadArrow + 0 + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 69 + + + + Class + LineGraphic + Head + + ID + 65 + + ID + 63 + Points + + {246.5, 275.5} + {246.5, 310} + + Style + + stroke + + HeadArrow + FilledArrow + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 64 + + + + Bounds + {{213.5, 261.5}, {66, 14}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 64 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 TCP, ZMQ } + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{169, 311}, {155, 60}} + Class + ShapedGraphic + ID + 65 + Shape + RoundRect + Style + + stroke + + Width + 2 + + + Text + + Align + 2 + RTFD + + BAtzdHJlYW10eXBlZIHoA4QBQISEhBJOU0F0dHJpYnV0 + ZWRTdHJpbmcAhIQITlNPYmplY3QAhZKEhIQITlNTdHJp + bmcBlIQBKwhMb2dzdGFzaIaEAmlJAQiShISEDE5TRGlj + dGlvbmFyeQCUhAFpBJKElpYOTlNPcmlnaW5hbEZvbnSG + koSEhAZOU0ZvbnQelJkohAVbNDBjXQYAAAAeAAAA//5W + AEEARwBSAG8AdQBuAGQAZQBkAEIAbwBsAGQAAACEAWYS + hAFjAJ0BnQCdAIaShJaWEE5TUGFyYWdyYXBoU3R5bGWG + koSEhBdOU011dGFibGVQYXJhZ3JhcGhTdHlsZQCEhBBO + U1BhcmFncmFwaFN0eWxlAJSEBENDQFMBAIUAhpKElpYG + TlNGb250hpKakoSWlgdOU0NvbG9yhpKEhIQHTlNDb2xv + cgCUnQKEBGZmZmaDlJMTP4Po5uY+g8rISD4BhoaG + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;\red128\green96\blue38;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qr + +\f0\b\fs36 \cf2 Logstash} + + + + Class + LineGraphic + Head + + ID + 68 + + ID + 66 + Points + + {246.5, 84.308083} + {246.5, 102.5} + + Style + + stroke + + HeadArrow + 0 + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 70 + + + + Class + LineGraphic + Head + + ID + 69 + + ID + 67 + Points + + {246.5, 130.5} + {246.5, 165} + + Style + + stroke + + HeadArrow + FilledArrow + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 68 + + + + Bounds + {{225.5, 102.5}, {42, 28}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 68 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 Traffic \ + Mirror } + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{169, 166}, {155, 60}} + Class + ShapedGraphic + ID + 69 + Shape + RoundRect + Style + + stroke + + Width + 2 + + + Text + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;\red255\green134\blue39;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs36 \cf2 nProbe} + + + + Bounds + {{208, 38.5}, {77, 46}} + Class + ShapedGraphic + ID + 70 + Shape + Cloud + Style + + stroke + + Width + 2 + + + Text + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 LAN} + VerticalPad + 0 + + + + GridInfo + + GuidesLocked + NO + GuidesVisible + YES + HPages + 1 + ImageCounter + 9 + ImageLinkBack + + + + + + ImageList + + image8.png + image7.png + image5.png + + KeepToScale + + Layers + + + Lock + NO + Name + Layer 1 + Print + YES + View + YES + + + LayoutInfo + + Animate + NO + circoMinDist + 18 + circoSeparation + 0.0 + layoutEngine + dot + neatoSeparation + 0.0 + twopiSeparation + 0.0 + + LinksVisible + NO + MagnetsVisible + NO + MasterSheets + + ModificationDate + 2014-02-05 01:14:27 +0000 + Modifier + Filippo Fontanelli + NotesVisible + NO + Orientation + 2 + OriginVisible + NO + PageBreaks + YES + PrintInfo + + NSBottomMargin + + float + 41 + + NSHorizonalPagination + + int + 0 + + NSLeftMargin + + float + 18 + + NSPaperSize + + coded + BAtzdHJlYW10eXBlZIHoA4QBQISEhAdOU1ZhbHVlAISECE5TT2JqZWN0AIWEASqEhAx7X05TU2l6ZT1mZn2WgVMCgUoDhg== + + NSPrintReverseOrientation + + int + 0 + + NSRightMargin + + float + 18 + + NSTopMargin + + float + 18 + + + PrintOnePage + + ReadOnly + NO + RowAlign + 1 + RowSpacing + 36 + SheetTitle + Canvas 1 + SmartAlignmentGuidesActive + YES + SmartDistanceGuidesActive + YES + UniqueID + 1 + UseEntirePage + + VPages + 1 + WindowInfo + + CurrentSheet + 0 + ExpandedCanvases + + + name + Canvas 1 + + + Frame + {{408, 103}, {929, 938}} + ListView + + OutlineWidth + 142 + RightSidebar + + ShowRuler + + Sidebar + + SidebarWidth + 120 + VisibleRegion + {{-118, 0}, {794, 783}} + Zoom + 1 + ZoomValues + + + Canvas 1 + 1 + 1 + + + + saveQuickLookFiles + YES + + diff --git a/splunk/doc/nprobe_elasticsearch_kibana.graffle/image5.png b/splunk/doc/nprobe_elasticsearch_kibana.graffle/image5.png new file mode 100644 index 0000000..0a64237 Binary files /dev/null and b/splunk/doc/nprobe_elasticsearch_kibana.graffle/image5.png differ diff --git a/splunk/doc/nprobe_elasticsearch_kibana.graffle/image7.png b/splunk/doc/nprobe_elasticsearch_kibana.graffle/image7.png new file mode 100644 index 0000000..dccd684 Binary files /dev/null and b/splunk/doc/nprobe_elasticsearch_kibana.graffle/image7.png differ diff --git a/splunk/doc/nprobe_elasticsearch_kibana.graffle/image8.png b/splunk/doc/nprobe_elasticsearch_kibana.graffle/image8.png new file mode 100644 index 0000000..02126e5 Binary files /dev/null and b/splunk/doc/nprobe_elasticsearch_kibana.graffle/image8.png differ diff --git a/splunk/doc/nprobe_splunk.graffle/data.plist b/splunk/doc/nprobe_splunk.graffle/data.plist new file mode 100644 index 0000000..3386d25 --- /dev/null +++ b/splunk/doc/nprobe_splunk.graffle/data.plist @@ -0,0 +1,669 @@ + + + + + ActiveLayerIndex + 0 + ApplicationVersion + + com.omnigroup.OmniGrafflePro + 138.33.0.157554 + + AutoAdjust + + BackgroundGraphic + + Bounds + {{0, 0}, {559, 783}} + Class + SolidGraphic + ID + 2 + Style + + shadow + + Draws + NO + + stroke + + Draws + NO + + + + CanvasOrigin + {0, 0} + ColumnAlign + 1 + ColumnSpacing + 36 + CreationDate + 2014-02-04 21:52:01 +0000 + Creator + Filippo Fontanelli + DisplayScale + 1 0/72 in = 1.0000 in + GraphDocumentVersion + 8 + GraphicsList + + + Bounds + {{343, 354}, {86, 42}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 72 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs24 \cf0 Search \ +& Storage \ +& Web Interface} + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{363, 205}, {46, 28}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 71 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs24 \cf0 Collector\ +Mode} + VerticalPad + 0 + + Wrap + NO + + + Class + LineGraphic + Head + + ID + 19 + + ID + 21 + Points + + {255, 250.00002} + {255, 280.5} + + Style + + stroke + + HeadArrow + 0 + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 11 + + + + Class + LineGraphic + Head + + ID + 24 + + ID + 20 + Points + + {255, 294.5} + {255, 344} + + Style + + stroke + + HeadArrow + FilledArrow + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 19 + + + + Bounds + {{238, 280.5}, {34, 14}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 19 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 TCP } + VerticalPad + 0 + + Wrap + NO + + + Class + Group + Graphics + + + Bounds + {{219.25, 361}, {71.5, 28}} + Class + ShapedGraphic + ID + 23 + ImageID + 1 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + + + Bounds + {{186, 345}, {138, 60}} + Class + ShapedGraphic + ID + 24 + Shape + RoundRect + Style + + stroke + + Width + 2 + + + + + ID + 22 + + + Class + LineGraphic + Head + + ID + 13 + + ID + 15 + Points + + {254.65947, 97.763466} + {254.18468, 129.5} + + Style + + stroke + + HeadArrow + 0 + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 9 + + + + Class + LineGraphic + Head + + ID + 11 + + ID + 14 + Points + + {254.82907, 157.5} + {254.45668, 188.00015} + + Style + + stroke + + HeadArrow + FilledArrow + LineType + 1 + TailArrow + 0 + Width + 2 + + + Tail + + ID + 13 + + + + Bounds + {{234, 129.5}, {42, 28}} + Class + ShapedGraphic + FitText + YES + Flow + Resize + ID + 13 + Shape + Rectangle + Style + + fill + + Draws + NO + + shadow + + Draws + NO + + stroke + + Draws + NO + + + Text + + Pad + 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 Traffic \ + Mirror } + VerticalPad + 0 + + Wrap + NO + + + Bounds + {{186, 189}, {138, 60}} + Class + ShapedGraphic + ID + 11 + Shape + RoundRect + Style + + stroke + + Width + 2 + + + Text + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 VAGRoundedBold;} +{\colortbl;\red255\green255\blue255;\red255\green134\blue39;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\b\fs36 \cf2 nProbe} + + + + Bounds + {{216.5, 52}, {77, 46}} + Class + ShapedGraphic + ID + 9 + Shape + Cloud + Style + + stroke + + Width + 2 + + + Text + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1265 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 LAN} + VerticalPad + 0 + + + + GridInfo + + GuidesLocked + NO + GuidesVisible + YES + HPages + 1 + ImageCounter + 2 + ImageLinkBack + + + + ImageList + + image1.png + + KeepToScale + + Layers + + + Lock + NO + Name + Layer 1 + Print + YES + View + YES + + + LayoutInfo + + Animate + NO + circoMinDist + 18 + circoSeparation + 0.0 + layoutEngine + dot + neatoSeparation + 0.0 + twopiSeparation + 0.0 + + LinksVisible + NO + MagnetsVisible + NO + MasterSheets + + ModificationDate + 2014-02-05 01:17:14 +0000 + Modifier + Filippo Fontanelli + NotesVisible + NO + Orientation + 2 + OriginVisible + NO + PageBreaks + YES + PrintInfo + + NSBottomMargin + + float + 41 + + NSHorizonalPagination + + int + 0 + + NSLeftMargin + + float + 18 + + NSPaperSize + + coded + BAtzdHJlYW10eXBlZIHoA4QBQISEhAdOU1ZhbHVlAISECE5TT2JqZWN0AIWEASqEhAx7X05TU2l6ZT1mZn2WgVMCgUoDhg== + + NSPrintReverseOrientation + + int + 0 + + NSRightMargin + + float + 18 + + NSTopMargin + + float + 18 + + + PrintOnePage + + ReadOnly + NO + RowAlign + 1 + RowSpacing + 36 + SheetTitle + Canvas 1 + SmartAlignmentGuidesActive + YES + SmartDistanceGuidesActive + YES + UniqueID + 1 + UseEntirePage + + VPages + 1 + WindowInfo + + CurrentSheet + 0 + ExpandedCanvases + + + name + Canvas 1 + + + Frame + {{435, -5}, {929, 938}} + ListView + + OutlineWidth + 142 + RightSidebar + + ShowRuler + + Sidebar + + SidebarWidth + 120 + VisibleRegion + {{-118, 0}, {794, 783}} + Zoom + 1 + ZoomValues + + + Canvas 1 + 1 + 1 + + + + saveQuickLookFiles + YES + + diff --git a/splunk/doc/nprobe_splunk.graffle/image1.png b/splunk/doc/nprobe_splunk.graffle/image1.png new file mode 100644 index 0000000..e0ff48d Binary files /dev/null and b/splunk/doc/nprobe_splunk.graffle/image1.png differ diff --git a/splunk/nprobe.spl b/splunk/nprobe.spl new file mode 100644 index 0000000..db89aac Binary files /dev/null and b/splunk/nprobe.spl differ diff --git a/splunk/nprobe/README.txt b/splunk/nprobe/README.txt new file mode 100644 index 0000000..36a1791 --- /dev/null +++ b/splunk/nprobe/README.txt @@ -0,0 +1,20 @@ +This free app is meant for: + +- Use with Splunk® Free +- Receive Flow data from nProbe + +This App from ntop.org allows you to ingest Flow data in Splunk® Free. + +Get started in 3 easy steps: + +1. Install and enable the app by completing the setup page +2. Send your Flow to the default listening port: 3333 via nProbe +3. Begin collecting and analyzing Flow with Splnk! + +nprobe -T "%IPV4_SRC_ADDR %L4_SRC_PORT %IPV4_DST_ADDR %L4_DST_PORT %PROTOCOL %IN_BYTES %OUT_BYTES %FIRST_SWITCHED %LAST_SWITCHED %HTTP_URL %HTTP_METHOD %HTTP_RET_CODE %HTTP_REFERER %HTTP_UA %HTTP_MIME %HTTP_HOST %HTTP_SITE %IN_PKTS %OUT_PKTS %IP_PROTOCOL_VERSION %APPLICATION_ID %L7_PROTO_NAME %ICMP_TYPE" --tcp "127.0.0.1:3333" -b 2 -i eth0 --json-labels + +Here is how it works: + +Flow is sent from your nProbe instance over the TCP protocol to port 3333. The nProbe bundled with the app receives the data and converts it into JSON messages. The data is sent to the local Splunk TCP data input 3333 which was created for you during the app installation. + +Please read the Quick Start Guide ($SPLUNK_HOME/ect/apps/nprobe/appserver/static). \ No newline at end of file diff --git a/splunk/nprobe/appserver/static/appIcon.png b/splunk/nprobe/appserver/static/appIcon.png new file mode 100644 index 0000000..e32bed8 Binary files /dev/null and b/splunk/nprobe/appserver/static/appIcon.png differ diff --git a/splunk/nprobe/appserver/static/nProbe Splunk App-QuickStart.pdf b/splunk/nprobe/appserver/static/nProbe Splunk App-QuickStart.pdf new file mode 100644 index 0000000..22d71f9 Binary files /dev/null and b/splunk/nprobe/appserver/static/nProbe Splunk App-QuickStart.pdf differ diff --git a/splunk/nprobe/appserver/static/screenshot.png b/splunk/nprobe/appserver/static/screenshot.png new file mode 100644 index 0000000..fa5c64a Binary files /dev/null and b/splunk/nprobe/appserver/static/screenshot.png differ diff --git a/splunk/nprobe/appserver/static/screenshot2.png b/splunk/nprobe/appserver/static/screenshot2.png new file mode 100644 index 0000000..472cac2 Binary files /dev/null and b/splunk/nprobe/appserver/static/screenshot2.png differ diff --git a/splunk/nprobe/bin/README b/splunk/nprobe/bin/README new file mode 100644 index 0000000..9a70db0 --- /dev/null +++ b/splunk/nprobe/bin/README @@ -0,0 +1 @@ +This is where you put any scripts you want to add to this app. diff --git a/splunk/nprobe/default/app.conf b/splunk/nprobe/default/app.conf new file mode 100644 index 0000000..254b7be --- /dev/null +++ b/splunk/nprobe/default/app.conf @@ -0,0 +1,20 @@ +# +# Splunk app configuration file +# + +[install] +is_configured = 0 +build = 1 + +[ui] +is_visible = 1 +label = nProbe + +[launcher] +author = ntop team +description = This application allows users to nProbe to export the flows information within Splunk. Once that is done, you can use the dashboards and reports already created to derive the standard information about the flow, host, protocols or create new report or dashboard to drive the information of your interest. +version = 1.2 + +[package] +check_for_updates = 1 +id = nprobe_app diff --git a/splunk/nprobe/default/data/models/nProbe.json b/splunk/nprobe/default/data/models/nProbe.json new file mode 100644 index 0000000..c8b08d7 --- /dev/null +++ b/splunk/nprobe/default/data/models/nProbe.json @@ -0,0 +1,244 @@ +{ + "objectSummary": { + "Interface Implementations": 0, + "Search-Based": 1, + "Transaction-Based": 0, + "Event-Based": 0, + "Interfaces": 0 + }, + "description": "", + "displayName": "nProbe", + "modelName": "nProbe", + "objects": [ + { + "parentName": "BaseSearch", + "objectName": "All", + "constraints": [], + "comment": "", + "calculations": [], + "displayName": "All", + "fields": [ + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "42", + "fieldName": "42", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "DST_AS", + "fieldName": "DST_AS", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "FIRST_SWITCHED", + "fieldName": "FIRST_SWITCHED", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "host", + "fieldName": "host", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "IN_BYTES", + "fieldName": "IN_BYTES", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "IN_PKTS", + "fieldName": "IN_PKTS", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "index", + "fieldName": "index", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "INPUT_SNMP", + "fieldName": "INPUT_SNMP", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "IPV4_DST_ADDR", + "fieldName": "IPV4_DST_ADDR", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "IPV4_DST_MASK", + "fieldName": "IPV4_DST_MASK", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "IPV4_NEXT_HOP", + "fieldName": "IPV4_NEXT_HOP", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "IPV4_SRC_ADDR", + "fieldName": "IPV4_SRC_ADDR", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "IPV4_SRC_MASK", + "fieldName": "IPV4_SRC_MASK", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "L4_DST_PORT", + "fieldName": "L4_DST_PORT", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "L4_SRC_PORT", + "fieldName": "L4_SRC_PORT", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "LAST_SWITCHED", + "fieldName": "LAST_SWITCHED", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "OUTPUT_SNMP", + "fieldName": "OUTPUT_SNMP", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "PROTOCOL", + "fieldName": "PROTOCOL", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "source", + "fieldName": "source", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "sourcetype", + "fieldName": "sourcetype", + "comment": "" + }, + { + "multivalue": false, + "type": "string", + "hidden": false, + "required": false, + "displayName": "splunk_server", + "fieldName": "splunk_server", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "SRC_AS", + "fieldName": "SRC_AS", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "SRC_TOS", + "fieldName": "SRC_TOS", + "comment": "" + }, + { + "multivalue": false, + "type": "number", + "hidden": false, + "required": false, + "displayName": "TCP_FLAGS", + "fieldName": "TCP_FLAGS", + "comment": "" + } + ], + "baseSearch": "index=nprobe" + } + ], + "objectNameList": [ + "All" + ] +} \ No newline at end of file diff --git a/splunk/nprobe/default/data/ui/nav/default.xml b/splunk/nprobe/default/data/ui/nav/default.xml new file mode 100644 index 0000000..f1da7ec --- /dev/null +++ b/splunk/nprobe/default/data/ui/nav/default.xml @@ -0,0 +1,39 @@ + diff --git a/splunk/nprobe/default/data/ui/views/README b/splunk/nprobe/default/data/ui/views/README new file mode 100644 index 0000000..6cf74f0 --- /dev/null +++ b/splunk/nprobe/default/data/ui/views/README @@ -0,0 +1 @@ +Add all the views that your app needs in this directory diff --git a/splunk/nprobe/default/data/ui/views/http.xml b/splunk/nprobe/default/data/ui/views/http.xml new file mode 100644 index 0000000..913580a --- /dev/null +++ b/splunk/nprobe/default/data/ui/views/http.xml @@ -0,0 +1,201 @@ +
+ + + +
+ + + + + + + + * + * + + + + * + * + * + + + + * + * + + + + * + * + + + + + @d + now + + +
+ + + Top site + `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)` | stats sum(Bytes) as Bytes by "Site Name" | sort -Bytes limit=30 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top Host + `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)`| stats sum(Bytes) as Bytes by "Source IP" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + Top Mine Type + HTTP_MIME!="" `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)`| stats sum(Bytes) as Bytes by HTTP_MIME + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top User Agent + HTTP_UA!="" `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)`| stats sum(Bytes) as Bytes by HTTP_UA + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Top ret code + HTTP_RET_CODE_NAME!="" `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)` | stats sum(Bytes) as Bytes by HTTP_RET_CODE_NAME + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top ret type + HTTP_RET_CODE_TYPE!="" `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)` | stats sum(Bytes) as Bytes by HTTP_RET_CODE_TYPE + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Top Referer + HTTP_REFERER!="" `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)` | stats sum(Bytes) as Bytes by HTTP_REFERER + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top method + HTTP_METHOD!="" `base_http($SRC_IP$,$SITE$,$CODE$,$TYPE$)` | stats sum(Bytes) as Bytes by HTTP_METHOD + $earliest$ + $latest$ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/splunk/nprobe/default/data/ui/views/overview.xml b/splunk/nprobe/default/data/ui/views/overview.xml new file mode 100644 index 0000000..3697a10 --- /dev/null +++ b/splunk/nprobe/default/data/ui/views/overview.xml @@ -0,0 +1,240 @@ +
+ + + +
+ + + + + + + + * + * + + + + * + + + + * + * + + + + + * + + + + * + + + + + @d + now + + +
+ + + Flows + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | timechart count as Flows + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + Top Flows + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | rename L4_SRC_PORT as "Source Port",L4_DST_PORT as "Destination Port", PROTOCOL_NAME as Protocol, SRC_HOST_NAME as "Source Name", DST_HOST_NAME as "Destination Name" | top limit=10 "Source IP","Source Name", "Source Port", "Destination IP","Destination Name", "Destination Port", Protocol, Bytes + $earliest$ + $latest$ + + + + + +
+ + + + Top 10 Source IP + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)`| stats sum(Bytes) as Bytes by "Source IP" + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + + + ["host","source","sourcetype"] + + + Top 10 Destination IP + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | stats sum(Bytes) as Bytes by "Destination IP" + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + + + ["host","source","sourcetype"] + + + + + Top Traffic Source Port + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)`|stats sum(Bytes) as Bytes by L4_SRC_PORT| sort -Bytes limit=10| rename L4_SRC_PORT as "Source Port" + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top Destination Port + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | stats sum(Bytes) as Bytes by L4_DST_PORT | sort -Bytes limit=10| rename L4_DST_PORT as "Destination Port" + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Top Protocols l4 + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)`|rename PROTOCOL_NAME as "Protocol Name"| stats sum(Bytes) as Bytes by "Protocol Name" + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top Protocols l7 + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` |rename L7_PROTO_NAME as "Protocol Name"| stats sum(Bytes) as Bytes by "Protocol Name" + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | stats sum(Bytes) + $earliest$ + $latest$ + + + + + + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | stats sum(Packets) + $earliest$ + $latest$ + + + + + + + `base_overview($PROTO_NAME$,$SRC_IP$,$SRC_PORT$,$DST_IP$,$DST_PORT$)` | stats count + $earliest$ + $latest$ + + + + + + \ No newline at end of file diff --git a/splunk/nprobe/default/data/ui/views/traffic_by_application.xml b/splunk/nprobe/default/data/ui/views/traffic_by_application.xml new file mode 100644 index 0000000..c3d2e98 --- /dev/null +++ b/splunk/nprobe/default/data/ui/views/traffic_by_application.xml @@ -0,0 +1,158 @@ +
+ + + +
+ + + + + + + + * + * + + + + * + * + + + + + @d + now + + +
+ + + Application per time + `base_traffic_application($PROTO_L4$,$APP$)` | rename L7_PROTO_NAME As "Protocol Name" +| timechart count as counter by "Protocol Name" | sort -counter + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + Top Application + `base_traffic_application($PROTO_L4$,$APP$)` | rename L7_PROTO_NAME As "Protocol Name" | stats sum(Bytes) as "Traffic" by "Protocol Name" | sort -Bytes limit=25 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Traffic by Source + `base_traffic_application($PROTO_L4$,$APP$)`|timechart sum(Bytes) by "Source IP" limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Traffic by Destination + `base_traffic_application($PROTO_L4$,$APP$)` |timechart sum(Bytes) by "Destination IP" limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + Top Source + `base_traffic_application($PROTO_L4$,$APP$)`|chart sum(Bytes) as "Sum of bytes" by "Source IP" | sort -Bytes limit=15 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top Source Ports + `base_traffic_application($PROTO_L4$,$APP$)`| rename L4_SRC_PORT as "Protocol Port" | stats sum(Bytes) as Bytes by "Protocol Port" | sort -Bytes limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/splunk/nprobe/default/data/ui/views/traffic_by_destination.xml b/splunk/nprobe/default/data/ui/views/traffic_by_destination.xml new file mode 100644 index 0000000..c4ff8f5 --- /dev/null +++ b/splunk/nprobe/default/data/ui/views/traffic_by_destination.xml @@ -0,0 +1,279 @@ +
+ + + +
+ + + + + + + + * + * + + + + * + + + + * + * + + + + + * + + + + * + + + + + @d + now + + +
+ + + Source Hosts per time + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| timechart count as counter by "Source IP" limit=10 | sort -counter + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Top 10 Source Hosts + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats sum(Bytes) as Bytes by "Source IP" | sort -Bytes limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + + + ["host","source","sourcetype"] + + + + + Top Host + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | rename IN_BYTES as Bytes,SRC_HOST_NAME as "Src Name", DST_HOST_NAME as "Dst Name" |top limit=10 "Source IP","Src Name", "Destination IP" , "Dst Name", Bytes | sort -Bytes,-percent, count + $earliest$ + $latest$ + + + + + +
+ + + + Top Protocols l4 + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | rename PROTOCOL_NAME as "Protocol Name" | stats sum(Bytes) as Bytes by "Protocol Name" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top Protocol l7 + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` |rename L7_PROTO_NAME as Application | stats sum(Bytes) as Bytes by Application | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Bytes distribution per source + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| stats sum(Bytes) as Bytes by "Source IP" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Average bytes per source + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats avg(IN_BYTES) as "Average Bytes" by "Source IP" | sort -"Average Bytes" limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + Packets distribution per source + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats sum(Packets) as Packets by "Source IP" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Average packets per Source + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats avg(Packets) as "Average Packets" by "Source IP" | sort -"Average Packets" limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + Last seen + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | top limit=20 _time + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + + + Top 10 source + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| top limit=20 source + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + +
+ + \ No newline at end of file diff --git a/splunk/nprobe/default/data/ui/views/traffic_by_source.xml b/splunk/nprobe/default/data/ui/views/traffic_by_source.xml new file mode 100644 index 0000000..fca2491 --- /dev/null +++ b/splunk/nprobe/default/data/ui/views/traffic_by_source.xml @@ -0,0 +1,278 @@ +
+ + + +
+ + + + + + + + * + * + + + + * + + + + * + * + + + + + * + + + + * + + + + + @d + now + + +
+ + + Destination Hosts per time + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| timechart count as counter by "Source IP" limit=10 | sort -counter + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Top 10 Destination Hosts + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats sum(Bytes) as Bytes by "Destination IP" | sort -Bytes limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + + + ["host","source","sourcetype"] + + + + + Top Host + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | rename IN_BYTES as Bytes,SRC_HOST_NAME as "Src Name", DST_HOST_NAME as "Dst Name" |top limit=10 "Source IP","Src Name", "Destination IP" , "Dst Name", Bytes | sort -Bytes,-percent,count + $earliest$ + $latest$ + + + + + +
+ + + + Top Protocols l4 + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` |rename PROTOCOL_NAME as "Protocol Name"| stats sum(Bytes) as Bytes by "Protocol Name" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Top Protocols l7 + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` |rename L7_PROTO_NAME as "Protocol Name"| stats sum(Bytes) as Bytes by "Protocol Name" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + Bytes distribution per destination + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats sum(Bytes) as Bytes by "Destination IP" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Average bytes per destination + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| stats avg(Bytes) as "Average Bytes" by "Destination IP" | sort -"Average Bytes" limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + Packets distribution per destination + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | stats sum(Packets) as Packets by "Destination IP" | sort -Bytes limit=20 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + Average packets per destination + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| stats avg(Packets) as "Average Packets" by "Destination IP" | sort -"Average Packets" limit=10 + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + Last seen + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)` | top limit=20 _time + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + + + + + + Top 10 source + `base_dest_src($PROTO_NAME$,"$SRC_IP$",$SRC_PORT$,"$DST_IP$",$DST_PORT$)`| top limit=20 source + $earliest$ + $latest$ + + + + + + + + + + + + + + + + + +
+ + \ No newline at end of file diff --git a/splunk/nprobe/default/datamodels.conf b/splunk/nprobe/default/datamodels.conf new file mode 100644 index 0000000..13ebba8 --- /dev/null +++ b/splunk/nprobe/default/datamodels.conf @@ -0,0 +1,2 @@ +[Ntopng] +acceleration = 0 diff --git a/splunk/nprobe/default/indexes.conf b/splunk/nprobe/default/indexes.conf new file mode 100644 index 0000000..d429e50 --- /dev/null +++ b/splunk/nprobe/default/indexes.conf @@ -0,0 +1,4 @@ +[nprobe] +coldPath = $SPLUNK_DB/nprobe/colddb +homePath = $SPLUNK_DB/nprobe/db +thawedPath = $SPLUNK_DB/nprobe/thaweddb diff --git a/splunk/nprobe/default/inputs.conf b/splunk/nprobe/default/inputs.conf new file mode 100644 index 0000000..cca7fc6 --- /dev/null +++ b/splunk/nprobe/default/inputs.conf @@ -0,0 +1,5 @@ +[tcp://3333] +source = nprobe-tcp +connection_host = dns +index = nprobe +sourcetype = json-too_small diff --git a/splunk/nprobe/default/macros.conf b/splunk/nprobe/default/macros.conf new file mode 100644 index 0000000..6f2622e --- /dev/null +++ b/splunk/nprobe/default/macros.conf @@ -0,0 +1,64 @@ +[source_host_name] +definition = eval "Source Host Name"= IPV4_SRC_ADDR."-".SRC_HOST_NAME +iseval = 0 + +[source_ip] +definition = rename IPV4_SRC_ADDR as "Source IP" +iseval = 0 + +[destination_host_name] +definition = eval "Destination Host Name"= IPV4_DST_ADDR."-".DST_HOST_NAME +iseval = 0 + +[destination_ip] +definition = rename IPV4_DST_ADDR as "Destination IP" +iseval = 0 + +[bytes] +definition = eval Bytes=IN_BYTES+OUT_BYTES +iseval = 0 + +[packets] +definition = eval Packets=IN_PKTS+OUT_PKTS +iseval = 0 + +[minimal_info] +definition = `bytes` | `packets` | `source_host_name` | `source_ip` | `destination_host_name` | `destination_ip` +iseval = 0 + +[base_overview(5)] +args = PROTO_NAME,SRC_IP, SRC_PORT, DST_IP, DST_PORT +definition = index=nprobe PROTOCOL_NAME=$PROTO_NAME$ IPV4_SRC_ADDR="$SRC_IP$" L4_SRC_PORT=$SRC_PORT$ IPV4_DST_ADDR="$DST_IP$" L4_DST_PORT=$DST_PORT$ |`minimal_info` +errormsg = Macro: error base_overview +iseval = 0 + +[base_traffic_application(2)] +args = PROTO_L4,APP +definition = index=nprobe PROTOCOL_NAME=$PROTO_L4$ L7_PROTO_NAME=$APP$ |`minimal_info` +errormsg = Macro: error base_traffic_application +iseval = 0 + +[base_dest_src(5)] +args = PROTO_NAME,SRC_IP, SRC_PORT, DST_IP, DST_PORT +definition = index=nprobe PROTOCOL_NAME=$PROTO_NAME$ IPV4_SRC_ADDR="$SRC_IP$" L4_SRC_PORT=$SRC_PORT$ IPV4_DST_ADDR="$DST_IP$" L4_DST_PORT=$DST_PORT$ |`minimal_info` +errormsg = Macro: error base_dest_src +iseval = 0 + +[base_http(4)] +args = SRC_IP,SITE,CODE,TYPE +definition = index=nprobe IPV4_SRC_ADDR="$SRC_IP$" HTTP_SITE="$SITE$" HTTP_RET_CODE_NAME="$CODE$" HTTP_RET_CODE_TYPE="$TYPE$" HTTP_RET_CODE_NAME!="Unknown" (HTTP_SITE!="" OR HTTP_HOST!="") |rename HTTP_SITE as "Site Name" |`minimal_info` +errormsg = Macro: error base_http +iseval = 0 + + +[bps] +definition = addinfo | eval TrafficSpeed = TrafficAmount*8 / (info_max_time - info_min_time) + +[pps] +definition = addinfo | eval PacketsSpeed = PacketsAmount / (info_max_time - info_min_time) + +[bps_chart] +definition = eval bps = ceil((bytes_in*8) / (t_int/1000)) + +[pps_chart] +definition = eval pps = ceil(packets_in / (t_int/1000)) \ No newline at end of file diff --git a/splunk/nprobe/default/props.conf b/splunk/nprobe/default/props.conf new file mode 100644 index 0000000..ba9e227 --- /dev/null +++ b/splunk/nprobe/default/props.conf @@ -0,0 +1,6 @@ +[json-too_small] +LOOKUP-protocols_lookups = protocol_lookups protocol AS PROTOCOL OUTPUTNEW protocol_name AS PROTOCOL_NAME +LOOKUP-http_ret_code_lookups = http_ret_code_lookups status AS HTTP_RET_CODE OUTPUTNEW status_description AS HTTP_RET_CODE_NAME status_type AS HTTP_RET_CODE_TYPE +LOOKUP-ip_dst_lookups = ip_lookups ip_address AS IPV4_DST_ADDR OUTPUTNEW host_name AS DST_HOST_NAME +LOOKUP-ip_src_lookups = ip_lookups ip_address AS IPV4_SRC_ADDR OUTPUTNEW host_name AS SRC_HOST_NAME + diff --git a/splunk/nprobe/default/transforms.conf b/splunk/nprobe/default/transforms.conf new file mode 100644 index 0000000..26702b9 --- /dev/null +++ b/splunk/nprobe/default/transforms.conf @@ -0,0 +1,17 @@ +[protocol_lookups] +default_match = Unknown +filename = protocols.csv +max_matches = 1 +min_matches = 0 + +[ip_lookups] +default_match = Unknown +filename = addressip.csv +max_matches = 1 +min_matches = 1 + +[http_ret_code_lookups] +default_match = Unknown +filename = http_ret_code.csv +max_matches = 1 +min_matches = 1 diff --git a/splunk/nprobe/lookups/addressip.csv b/splunk/nprobe/lookups/addressip.csv new file mode 100644 index 0000000..5b098a0 --- /dev/null +++ b/splunk/nprobe/lookups/addressip.csv @@ -0,0 +1,3 @@ +ip_address,host_name +192.168.1.129,ntop_server +192.168.1.133,MacBookPro diff --git a/splunk/nprobe/lookups/http_ret_code.csv b/splunk/nprobe/lookups/http_ret_code.csv new file mode 100644 index 0000000..2311819 --- /dev/null +++ b/splunk/nprobe/lookups/http_ret_code.csv @@ -0,0 +1,41 @@ +status,status_description,status_type +100,Continue,Informational +101,Switching Protocols,Informational +200,OK,Successful +201,Created,Successful +202,Accepted,Successful +203,Non-Authoritative Information,Successful +204,No Content,Successful +205,Reset Content,Successful +206,Partial Content,Successful +300,Multiple Choices,Redirection +301,Moved Permanently,Redirection +302,Found,Redirection +303,See Other,Redirection +304,Not Modified,Redirection +305,Use Proxy,Redirection +307,Temporary Redirect,Redirection +400,Bad Request,Client Error +401,Unauthorized,Client Error +402,Payment Required,Client Error +403,Forbidden,Client Error +404,Not Found,Client Error +405,Method Not Allowed,Client Error +406,Not Acceptable,Client Error +407,Proxy Authentication Required,Client Error +408,Request Timeout,Client Error +409,Conflict,Client Error +410,Gone,Client Error +411,Length Required,Client Error +412,Precondition Failed,Client Error +413,Request Entity Too Large,Client Error +414,Request-URI Too Long,Client Error +415,Unsupported Media Type,Client Error +416,Requested Range Not Satisfiable,Client Error +417,Expectation Failed,Client Error +500,Internal Server Error,Server Error +501,Not Implemented,Server Error +502,Bad Gateway,Server Error +503,Service Unavailable,Server Error +504,Gateway Timeout,Server Error +505,HTTP Version Not Supported,Server Error \ No newline at end of file diff --git a/splunk/nprobe/lookups/icmp.csv b/splunk/nprobe/lookups/icmp.csv new file mode 100644 index 0000000..36bc427 --- /dev/null +++ b/splunk/nprobe/lookups/icmp.csv @@ -0,0 +1,39 @@ +Type,Name,Reference +0,Echo Reply,[RFC792] +1,Unassigned, +2,Unassigned, +3,Destination Unreachable,[RFC792] +4,Source Quench (Deprecated),[RFC792][RFC6633] +5,Redirect,[RFC792] +6,Alternate Host Address (Deprecated),[RFC6918] +7,Unassigned, +8,Echo,[RFC792] +9,Router Advertisement,[RFC1256] +10,Router Solicitation,[RFC1256] +11,Time Exceeded,[RFC792] +12,Parameter Problem,[RFC792] +13,Timestamp,[RFC792] +14,Timestamp Reply,[RFC792] +15,Information Request (Deprecated),[RFC792][RFC6918] +16,Information Reply (Deprecated),[RFC792][RFC6918] +17,Address Mask Request (Deprecated),[RFC950][RFC6918] +18,Address Mask Reply (Deprecated),[RFC950][RFC6918] +19,Reserved (for Security),[Solo] +20-29,Reserved (for Robustness Experiment),[ZSu] +30,Traceroute (Deprecated),[RFC1393][RFC6918] +31,Datagram Conversion Error (Deprecated),[RFC1475][RFC6918] +32,Mobile Host Redirect (Deprecated),[David_Johnson][RFC6918] +33,IPv6 Where-Are-You (Deprecated),[Simpson][RFC6918] +34,IPv6 I-Am-Here (Deprecated),[Simpson][RFC6918] +35,Mobile Registration Request (Deprecated),[Simpson][RFC6918] +36,Mobile Registration Reply (Deprecated),[Simpson][RFC6918] +37,Domain Name Request (Deprecated),[RFC1788][RFC6918] +38,Domain Name Reply (Deprecated),[RFC1788][RFC6918] +39,SKIP (Deprecated),[Markson][RFC6918] +40,Photuris,[RFC2521] +41,"ICMP messages utilized by experimental + mobility protocols such as Seamoby",[RFC4065] +42-252,Unassigned, +253,RFC3692-style Experiment 1,[RFC4727] +254,RFC3692-style Experiment 2,[RFC4727] +255,Reserved,[JBP] \ No newline at end of file diff --git a/splunk/nprobe/lookups/protocols.csv b/splunk/nprobe/lookups/protocols.csv new file mode 100644 index 0000000..3a5be2d --- /dev/null +++ b/splunk/nprobe/lookups/protocols.csv @@ -0,0 +1,131 @@ +protocol,protocol_name +0,ip +1,icmp +2,igmp +3,ggp +4,ipencap +5,st2 +6,tcp +7,cbt +8,egp +9,igp +10,bbn-rcc +11,nvp +12,pup +13,argus +14,emcon +15,xnet +16,chaos +17,udp +18,mux +19,dcn +20,hmp +21,prm +22,xns-idp +23,trunk-1 +24,trunk-2 +25,leaf-1 +26,leaf-2 +27,rdp +28,irtp +29,iso-tp4 +30,netblt +31,mfe-nsp +32,merit-inp +33,sep +34,3pc +35,idpr +36,xtp +37,ddp +38,idpr-cmtp +39,tp++ +40,il +41,ipv6 +42,sdrp +43,ipv6-route +44,ipv6-frag +45,idrp +46,rsvp +47,gre +48,mhrp +49,bna +50,esp +51,ah +52,i-nlsp +53,swipe +54,narp +55,mobile +56,tlsp +57,skip +58,ipv6-icmp +59,ipv6-nonxt +60,ipv6-opts +62,cftp +64,sat-expak +65,kryptolan +66,rvd +67,ippc +69,sat-mon +70,visa +71,ipcv +72,cpnx +73,cphb +74,wsn +75,pvp +76,br-sat-mon +77,sun-nd +78,wb-mon +79,wb-expak +80,iso-ip +81,vmtp +82,secure-vmtp +83,vines +84,ttp +85,nsfnet-igp +86,dgp +87,tcf +88,eigrp +89,ospf +90,sprite-rpc +91,larp +92,mtp +93,ax.25 +94,ipip +95,micp +96,scc-sp +97,etherip +98,encap +100,gmtp +101,ifmp +102,pnni +103,pim +104,aris +105,scps +106,qnx +107,a/n +108,ipcomp +109,snp +110,compaq-peer +111,ipx-in-ip +112,vrrp +113,pgm +115,l2tp +116,ddx +117,iatp +118,st +119,srp +120,uti +121,smp +122,sm +123,ptp +124,isis +125,fire +126,crtp +127,crdup +128,sscopmce +129,iplt +130,sps +131,pipe +132,sctp +133,fc +254,divert diff --git a/splunk/nprobe/metadata/default.meta b/splunk/nprobe/metadata/default.meta new file mode 100644 index 0000000..d2b76c9 --- /dev/null +++ b/splunk/nprobe/metadata/default.meta @@ -0,0 +1,113 @@ + +# Application-level permissions + +[] +access = read : [ * ], write : [ admin ] +export = none + + + +### EVENT TYPES + +[eventtypes] +export = system +access = read : [ * ], write : [ power ] + +### PROPS + +[props] +export = system + + +### TRANSFORMS + +[transforms] +export = system + +[transforms/http_ret_code_lookups] +export = none +owner = admin + +[transforms/protocol_lookups] +export = none +owner = admin + + +[transforms/ip_lookups] +export = none +owner = admin + +### LOOKUPS + +[lookups] +export = system + +[props/json-too_small/LOOKUP-protocols_lookups] +export = none +owner = admin + +[props/json-too_small/LOOKUP-http_ret_code_lookups] +export = none +owner = admin + + +[lookups/protocols.csv] +access = read : [ * ], write : [ admin, power ] +export = none +owner = admin + + + +### VIEWSTATES: even normal users should be able to create shared viewstates + +[viewstates] +access = read : [ * ], write : [ * ] +export = system + +[views/traffic_by_source] +owner = admin + + +[views/traffic_by_destination] +owner = admin + + +[views/traffic_by_application] +owner = admin + + +[views/http] +access = read : [ * ], write : [ admin ] +export = none +owner = admin + + +[views/overview] +access = read : [ * ], write : [ admin ] +export = none +owner = admin + + +### MACROS + +[macros/source_host_name] +access = read : [ * ], write : [ admin ] +export = none + + +[macros/destination_host_name] +access = read : [ * ], write : [ admin ] +export = none + + +[macros/bytes] +access = read : [ * ], write : [ admin ] +export = none + + +[macros/packets] +access = read : [ * ], write : [ admin ] +export = none + + + diff --git a/zmq/README b/zmq/README new file mode 100644 index 0000000..b46e31b --- /dev/null +++ b/zmq/README @@ -0,0 +1,3 @@ +Code examples of subscriber/publisher that can be used with ZMQ. + +nProbe can export data via ZMQ and the subscriber can be used as skeleton/testing tool for playing with flow-data received via ZMQ. \ No newline at end of file diff --git a/zmq/zmq_publisher/Makefile b/zmq/zmq_publisher/Makefile new file mode 100644 index 0000000..32909b5 --- /dev/null +++ b/zmq/zmq_publisher/Makefile @@ -0,0 +1,10 @@ +CC=gcc -I /usr/local/include +CFLAGS= -g -L /usr/local/lib -lzmq + +all: publisher + +publisher: publisher.c + $(CC) $^ -o $@ $(CFLAGS) + +clean: + rm -f publisher diff --git a/zmq/zmq_publisher/publisher.c b/zmq/zmq_publisher/publisher.c new file mode 100644 index 0000000..839b86b --- /dev/null +++ b/zmq/zmq_publisher/publisher.c @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MSG_VERSION 0 + +/* ***************************************************************** */ + +struct zmq_msg_hdr { + char url[32]; + u_int32_t version; + u_int32_t size; +}; + +/* ***************************************************************** */ + +void help() { + printf("publisher \n"); + exit(0); +} + +/* ***************************************************************** */ + + +int main(int argc, char *argv []) { + void *context = zmq_ctx_new(); + void *publisher = zmq_socket(context, ZMQ_PUB); + int rc, size; + struct zmq_msg_hdr msg_hdr; + char *endpoint = "tcp://127.0.0.1:5556"; + char *topic = "flow", buffer[512]; + FILE *fd; + + if(argc != 2) help(); + fd = fopen(argv[1], "r"); + + if(fd == NULL) { + printf("Unable to open file %s\n", argv[1]); + exit(0); + } + + rc = zmq_bind(publisher, endpoint); + if(rc != 0) { + printf("Unable to bind to %s\n", endpoint); + return -1; + } else + printf("Publishing data to %s...\n", endpoint); + + printf("Waiting ntopng to start...\n"); + sleep(3); + + printf("Ready\n"); + + msg_hdr.version = 0, snprintf(msg_hdr.url, sizeof(msg_hdr.url), "flow"); + + while(fgets(buffer, sizeof(buffer), fd) != NULL) { + // |00:10:07|joe|10.49.133.251|53651|205.188.250.75|993|0| + + printf("%s", buffer); + if(buffer[0] == '|') { + char flow[2048]; + int h, m, s; + u_int32_t begin, end; + char *when, *user, *src_ip, *dst_ip; + char *sport, *dport, *proto; + + when = strtok(buffer, "|"); + user = strtok(NULL, "|"); + src_ip = strtok(NULL, "|"); + sport = strtok(NULL, "|"); + dst_ip = strtok(NULL, "|"); + dport = strtok(NULL, "|"); + proto = strtok(NULL, "|"); + + sscanf(when, "%u:%u:%u", &h, &m, &s); + begin = time(NULL); + begin -= (begin % 86400); + begin += h*3600+m*60+s; + end = begin + 60 /* 1 min */; + snprintf(flow, sizeof(flow), + "{\"8\":\"%s\",\"12\":\"%s\",\"7\":%s,\"11\":%s,\"4\":%s,\"22\":%u,\"21\":%u}", + src_ip, dst_ip, sport, dport, proto, begin, end); + + msg_hdr.size = strlen(flow); + zmq_send(publisher, &msg_hdr, sizeof(msg_hdr), ZMQ_SNDMORE); + zmq_send(publisher, flow, msg_hdr.size, 0); + } + } + + fclose(fd); + zmq_close(publisher); + zmq_ctx_destroy(context); + + return 0; +} + diff --git a/zmq/zmq_publisher/testfile.txt b/zmq/zmq_publisher/testfile.txt new file mode 100644 index 0000000..bc88b1c --- /dev/null +++ b/zmq/zmq_publisher/testfile.txt @@ -0,0 +1,99 @@ +#time|username|srcip|srcport|dstip|dstport|protocol| +|00:10:07|joe|10.49.133.251|53651|205.188.250.75|993|0| +|00:10:24|joe|10.49.133.251|53652|17.151.226.33|443|0| +|00:10:25|joe|10.49.133.251|53653|17.151.226.33|443|0| +|00:30:43|joe|10.49.133.251|53656|205.188.157.116|993|0| +|00:31:01|joe|10.49.133.251|53658|17.151.226.32|443|0| +|00:31:01|joe|10.49.133.251|53659|17.151.226.32|443|0| +|00:54:55|joe|10.49.133.251|68|10.49.48.10|67|1| +|01:48:18|joe|10.49.133.251|53667|205.188.170.17|993|0| +|01:48:34|joe|10.49.133.251|53668|17.151.226.32|443|0| +|01:48:34|joe|10.49.133.251|53669|17.151.226.32|443|0| +|02:10:29|joe|10.49.133.251|53673|64.12.95.121|993|0| +|02:30:09|joe|10.49.133.251|53677|205.188.159.89|993|0| +|02:30:10|joe|10.49.133.251|53678|64.12.78.57|993|0| +|02:30:26|joe|10.49.133.251|53679|17.151.226.32|443|0| +|02:30:26|joe|10.49.133.251|53680|17.151.226.32|443|0| +|13:04:05|joe|10.49.133.251|50203|159.45.2.36|443|0| +|13:04:05|joe|10.49.133.251|50202|159.45.2.36|443|0| +|13:05:11|joe|10.49.133.251|60116|74.125.239.48|80|0| +|13:05:11|joe|10.49.133.251|60117|74.125.239.48|443|0| +|13:05:13|joe|10.49.133.251|60118|205.188.27.128|80|0| +|13:05:13|joe|10.49.133.251|49665|207.200.74.12|80|0| +|13:05:13|joe|10.49.133.251|49666|207.200.74.12|443|0| +|13:05:13|joe|10.49.133.251|49667|64.12.66.245|80|0| +|13:05:13|joe|10.49.133.251|49668|64.12.66.245|80|0| +|13:05:13|joe|10.49.133.251|49669|64.12.96.192|80|0| +|13:05:13|joe|10.49.133.251|49811|64.12.132.39|80|0| +|13:05:13|joe|10.49.133.251|49812|64.12.132.39|80|0| +|13:05:13|joe|10.49.133.251|49813|165.254.99.83|80|0| +|13:05:13|joe|10.49.133.251|49814|165.254.99.83|80|0| +|13:05:13|joe|10.49.133.251|49815|64.12.201.118|80|0| +|13:05:13|joe|10.49.133.251|49816|207.200.74.71|80|0| +|13:05:13|joe|10.49.133.251|49817|63.140.35.162|80|0| +|13:05:13|joe|10.49.133.251|49818|54.243.114.132|443|0| +|13:05:13|joe|10.49.133.251|61495|207.200.74.53|80|0| +|13:05:13|joe|10.49.133.251|52461|192.80.13.26|80|0| +|13:05:14|joe|10.49.133.251|59981|207.200.81.19|80|0| +|13:05:14|joe|10.49.133.251|59982|165.254.47.105|80|0| +|13:05:14|joe|10.49.133.251|59983|23.20.64.198|80|0| +|13:05:14|joe|10.49.133.251|57816|165.254.99.96|80|0| +|13:05:14|joe|10.49.133.251|55917|165.254.47.128|80|0| +|13:05:14|joe|10.49.133.251|55918|207.200.81.19|80|0| +|13:05:14|joe|10.49.133.251|55919|207.200.81.13|80|0| +|13:05:14|joe|10.49.133.251|55920|74.121.136.104|80|0| +|13:05:14|joe|10.49.133.251|55921|199.38.166.165|80|0| +|13:05:14|joe|10.49.133.251|55922|207.200.81.20|80|0| +|13:05:14|joe|10.49.133.251|55923|74.125.239.59|80|0| +|13:05:14|joe|10.49.133.251|55924|204.2.197.201|80|0| +|13:05:14|joe|10.49.133.251|55925|207.200.74.76|80|0| +|13:05:14|joe|10.49.133.251|55926|74.125.239.123|80|0| +|13:05:14|joe|10.49.133.251|55927|207.200.74.76|80|0| +|13:05:14|joe|10.49.133.251|55928|205.188.100.51|80|0| +|13:05:14|joe|10.49.133.251|55929|68.67.128.40|80|0| +|13:05:14|joe|10.49.133.251|54024|207.200.81.20|80|0| +|13:05:14|joe|10.49.133.251|54025|207.200.81.20|80|0| +|13:05:17|joe|10.49.133.251|54026|207.200.74.12|443|0| +|13:05:18|joe|10.49.133.251|54027|64.12.66.245|80|0| +|13:05:19|joe|10.49.133.251|61236|54.225.39.252|843|0| +|13:05:19|joe|10.49.133.251|61237|165.254.47.128|80|0| +|13:05:20|joe|10.49.133.251|61238|54.225.39.252|80|0| +|13:05:20|joe|10.49.133.251|61239|54.225.39.252|80|0| +|13:05:20|joe|10.49.133.251|61240|54.225.39.252|80|0| +|13:05:21|joe|10.49.133.251|61241|54.225.39.252|80|0| +|13:05:24|joe|10.49.133.251|61242|54.225.39.252|80|0| +|13:05:26|joe|10.49.133.251|61243|54.225.39.252|80|0| +|13:05:26|joe|10.49.133.251|61244|207.200.81.13|80|0| +|13:05:27|joe|10.49.133.251|61245|68.67.128.40|80|0| +|13:05:27|joe|10.49.133.251|61246|204.2.197.201|80|0| +|13:05:27|joe|10.49.133.251|61247|207.200.74.76|443|0| +|13:05:30|joe|10.49.133.251|63833|107.21.234.205|443|0| +|13:05:30|joe|10.49.133.251|63834|63.149.195.18|80|0| +|13:05:31|joe|10.49.133.251|63835|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63836|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63837|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63838|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63839|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63840|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63841|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63842|208.50.56.240|80|0| +|13:05:31|joe|10.49.133.251|63843|199.7.54.72|80|0| +|13:05:34|joe|10.49.133.251|63844|23.22.254.199|80|0| +|13:05:35|joe|10.49.133.251|63845|23.22.254.199|80|0| +|13:05:37|joe|10.49.133.251|63846|23.22.254.199|80|0| +|13:05:39|joe|10.49.133.251|63847|23.22.254.199|80|0| +|13:05:41|joe|10.49.133.251|63848|23.22.254.199|80|0| +|13:05:41|joe|10.49.133.251|63849|63.140.35.162|80|0| +|13:05:43|joe|10.49.133.251|63850|23.22.254.199|80|0| +|13:05:45|joe|10.49.133.251|63851|23.22.254.199|80|0| +|04:28:31|joe|10.49.133.251|53691|205.188.170.17|993|0| +|13:09:13|joe|10.49.133.251|58917|184.84.222.18|80|0| +|13:09:29|joe|10.49.133.251|58919|17.172.233.126|5223|0| +|05:10:21|joe|10.49.133.251|53697|205.188.250.75|993|0| +|05:10:40|joe|10.49.133.251|53698|17.151.226.32|443|0| +|05:43:45|joe|10.49.133.251|53703|149.174.159.97|993|0| +|05:44:03|joe|10.49.133.251|53705|17.151.226.32|443|0| +|05:44:03|joe|10.49.133.251|53706|17.151.226.32|443|0| +|06:00:53|joe|10.49.133.251|53710|205.188.170.17|993|0| +|06:01:14|joe|10.49.133.251|53712|17.151.226.33|443|0| +|06:24:32|joe|10.49.133.251|53717|205.188.155.221|993|0| diff --git a/zmq/zmq_subscriber/Makefile b/zmq/zmq_subscriber/Makefile new file mode 100644 index 0000000..6d43ba8 --- /dev/null +++ b/zmq/zmq_subscriber/Makefile @@ -0,0 +1,16 @@ +CC=gcc +CFLAGS= -g -I/usr/local/include +LIBS=-L/usr/local/lib -lzmq +OS := $(shell uname -s) + +ifeq ($(OS),Darwin) +CFLAGS +=-fno-color-diagnostics +endif + +all: subscriber + +subscriber: subscriber.c + $(CC) $(CFLAGS) $^ -o $@ $(LIBS) + +clean: + rm -f subscriber diff --git a/zmq/zmq_subscriber/subscriber.c b/zmq/zmq_subscriber/subscriber.c new file mode 100644 index 0000000..6ee530f --- /dev/null +++ b/zmq/zmq_subscriber/subscriber.c @@ -0,0 +1,171 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TOPIC "flow" + +#define MSG_VERSION 0 + +struct zmq_msg_hdr { + char url[32]; + u_int32_t version; + u_int32_t size; +}; + +void help() { + printf("subscriber -i [-v] [-t ][-f ]\n\n\n" + "Example: subscriber -i tcp://127.0.0.1:5556 -f 192.168.1.1:1234\n\n" + "Subscribe to ZMQ events generated by tcp://127.0.0.1:5556 and forward them\n" + "via TCP to 192.168.1.1:1234\n" + ); + + exit(0); +} + +int main (int argc, char *argv []) { + void *context = zmq_ctx_new(); + void *subscriber = zmq_socket(context, ZMQ_SUB); + char *payload, c; + int payload_len, verbose = 0; + int rc, size; + struct zmq_msg_hdr h; + char *iface = NULL; // "tcp://127.0.0.1:5556"; + char *topic = TOPIC; + char *forward_host = NULL; + int tcp_socket = -1; + struct sockaddr_in servaddr; + + while((c = getopt(argc, argv, "i:t:hf:v")) != -1) { + switch(c) { + case 'f': + forward_host = optarg; + break; + case 'i': + iface = optarg; + break; + case 't': + topic = optarg; + break; + case 'v': + verbose = 1; + break; + default: + help(); + } + } + + if((iface == NULL) || (topic == NULL)) + help(); + + redo: + if(forward_host) { + char buf[256], *host, *port; + struct hostent *server; + + snprintf(buf, sizeof(buf), "%s", forward_host); + + host = strtok(buf, ":"); + if(!host) help(); + + port = strtok(NULL, ":"); + if(!port) help(); + + if(!(server = gethostbyname(host))) { + printf("Unknown host %s\n", host); + help(); + } + + tcp_socket = socket(AF_INET, SOCK_STREAM, 0); + + bzero(&servaddr, sizeof(servaddr)); + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = (*(struct in_addr *)server->h_addr_list[0]).s_addr; + servaddr.sin_port=htons(atoi(port)); + + if((rc = connect(tcp_socket, (struct sockaddr *)&servaddr, sizeof(servaddr))) != 0) { + printf("Unable to connect to %s:%s [%s/%u]\n", host, port, strerror(errno), errno); + help(); + } else + printf("Connected to %s:%s\n", host, port); + } + + rc = zmq_connect(subscriber, iface); + + if (rc != 0) + return -1; + + if(strcmp(topic, "both") == 0) { + char *t; + + t = "event"; rc = zmq_setsockopt(subscriber, ZMQ_SUBSCRIBE, t, strlen(t)); + t = "flow"; rc = zmq_setsockopt(subscriber, ZMQ_SUBSCRIBE, t, strlen(t)); + } else + rc = zmq_setsockopt(subscriber, ZMQ_SUBSCRIBE, topic, strlen(topic)); + + if (rc != 0) + return -1; + + printf("Listening for %s on %s\n", topic, iface); + + while (1) { + char theDate[32]; + time_t theTime; + struct zmq_msg_hdr msg_hdr; + + size = zmq_recv(subscriber, &h, sizeof(h), 0); + + if (size != sizeof(h) || h.version != MSG_VERSION) { + printf("Unsupported publisher version\n"); + return -1; + } + + payload_len = h.size + 2; + payload = malloc(payload_len); + + size = zmq_recv(subscriber, payload, payload_len, 0); + + if(tcp_socket != -1) { + payload[h.size] = '\n'; + payload[h.size+1] = '\0'; + rc = write(tcp_socket, payload, (int)(h.size+1)); + + if(rc < 0) { + printf("Send error %d [%s/%u]\n", tcp_socket, strerror(errno), errno); + zmq_close(subscriber); + zmq_ctx_destroy(context); + close(tcp_socket); + sleep(1); + goto redo; + break; + } + + printf("%s", payload); + } + + if(verbose) { + time_t theTime = time(NULL); + + payload[h.size] = '\0'; + + strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime(&theTime)); + printf("[%s] %s\n", theDate, payload); + } + + free(payload); + } + + zmq_close(subscriber); + zmq_ctx_destroy(context); + + return 0; +} +