Skip to content
Permalink
Browse files

restructured commandline tools

  • Loading branch information...
dreadl0ck committed May 3, 2019
1 parent 461e3e8 commit e777c51edb397c6abee58bdbe54782db4b082245
Showing with 276 additions and 149 deletions.
  1. +16 −35 cmd/capture/flags.go
  2. +2 −57 cmd/capture/main.go
  3. +0 −55 cmd/capture/utils.go
  4. +33 −0 cmd/dump/flags.go
  5. +64 −1 cmd/dump/main.go
  6. +35 −0 cmd/export/flags.go
  7. +113 −1 cmd/export/main.go
  8. +13 −0 cmd/label/main.go
@@ -18,51 +18,32 @@ import (
)

var (
flagBPF = flag.String("bpf", "", "supply a BPF filter to use prior to processing packets with netcap")
flagInclude = flag.String("include", "", "include specific encoders")
flagExclude = flag.String("exclude", "", "exclude specific encoders")
flagEncoders = flag.Bool("encoders", false, "show all available encoders")
flagInput = flag.String("r", "", "read specified file, can either be a pcap or netcap audit record file")
flagInput = flag.String("r", "", "read specified file, can either be a pcap or netcap audit record file")
flagOutDir = flag.String("out", "", "specify output directory, will be created if it does not exist")

flagBPF = flag.String("bpf", "", "supply a BPF filter to use prior to processing packets with netcap")

flagInclude = flag.String("include", "", "include specific encoders")
flagExclude = flag.String("exclude", "", "exclude specific encoders")

flagEncoders = flag.Bool("encoders", false, "show all available encoders")
flagPrintProtocolOverview = flag.Bool("overview", false, "print a list of all available encoders and fields")

flagInterface = flag.String("iface", "", "attach to network interface and capture in live mode")
flagSelect = flag.String("select", "", "select specific fields of an audit records when generating csv or tables")
flagFields = flag.Bool("fields", false, "print available fields for an audit record file and exit")
flagCompress = flag.Bool("comp", true, "compress output with gzip")
flagBuffer = flag.Bool("buf", true, "buffer data in memory before writing to disk")
flagWorkers = flag.Int("workers", 1000, "number of workers")
flagSeparator = flag.String("sep", ",", "set separator string for csv output")
flagPacketBuffer = flag.Int("pbuf", 100, "set packet buffer size, for channels that feed data to workers")
flagOutDir = flag.String("out", "", "specify output directory, will be created if it does not exist")

flagCPUProfile = flag.Bool("cpuprof", false, "create cpu profile")
flagMemProfile = flag.Bool("memprof", false, "create memory profile")

flagUTC = flag.Bool("utc", false, "print timestamps as UTC when using select csv")
flagToUTC = flag.String("ts2utc", "", "util to convert sencods.microseconds timestamp to UTC")

flagCSV = flag.Bool("csv", false, "print output data as csv with header line")
flagPrintStructured = flag.Bool("struc", false, "print output as structured objects")
flagTSV = flag.Bool("tsv", false, "print output as tab separated values")

flagCPUProfile = flag.Bool("cpuprof", false, "create cpu profile")
flagMemProfile = flag.Bool("memprof", false, "create memory profile")
flagIngoreUnknown = flag.Bool("ignore-unknown", false, "disable writing unknown packets into a pcap file")
flagPromiscMode = flag.Bool("promisc", true, "toggle promiscous mode for live capture")
flagSnapLen = flag.Int("snaplen", 1024, "configure snaplen for live capture from interface")

flagHeader = flag.Bool("header", false, "print audit record file header and exit")
flagVersion = flag.Bool("version", false, "print netcap package version and exit")
flagTable = flag.Bool("table", false, "print output as table view (thanks @evilsocket)")
flagCheckFields = flag.Bool("check", false, "check number of occurences of the separator, in fields of an audit record file")

flagPromiscMode = flag.Bool("promisc", true, "toggle promiscous mode for live capture")
flagSnapLen = flag.Int("snaplen", 1024, "configure snaplen for live capture from interface")
flagPrintProtocolOverview = flag.Bool("overview", false, "print a list of all available encoders and fields")
flagVersion = flag.Bool("version", false, "print netcap package version and exit")

flagBaseLayer = flag.String("base", "ethernet", "select base layer")
flagDecodeOptions = flag.String("opts", "lazy", "select decoding options")
flagPayload = flag.Bool("payload", false, "capture payload for supported layers")

flagBegin = flag.String("begin", "(", "begin character for a structure in CSV output")
flagEnd = flag.String("end", ")", "end character for a structure in CSV output")
flagStructSeparator = flag.String("struct-sep", "-", "separator character for a structure in CSV output")

// move to exporter?
flagExport = flag.Bool("export", true, "export prometheus metrics")
flagMetricsAddress = flag.String("address", "127.0.0.1:4444", "metrics address")
)
@@ -18,19 +18,15 @@ import (
"fmt"
"log"
"os"
"path/filepath"
"runtime/pprof"
"strconv"
"time"

"github.com/dreadl0ck/netcap/metrics"

"github.com/mgutz/ansi"

"github.com/dreadl0ck/netcap"
"github.com/dreadl0ck/netcap/collector"
"github.com/dreadl0ck/netcap/encoder"
"github.com/dreadl0ck/netcap/types"
"github.com/dreadl0ck/netcap/utils"
"github.com/evilsocket/islazy/tui"
)
@@ -54,12 +50,6 @@ func main() {
return
}

// util to convert netcap timestamp to UTC time
if *flagToUTC != "" {
fmt.Println(utils.TimeToUTC(*flagToUTC))
os.Exit(1)
}

// configure CPU profiling
if *flagCPUProfile {
defer func() func() {
@@ -102,36 +92,6 @@ func main() {
os.Exit(1)
}

// util to check if fields count matches for all generated rows
if *flagCheckFields {
checkFields()
return
}

// read dumpfile header and exit
if *flagHeader {

// open input file for reading
r, err := netcap.Open(*flagInput)
if err != nil {
panic(err)
}

// get header
// this will panic if the header is corrupted
h := r.ReadHeader()

// print result as table
tui.Table(os.Stdout, []string{"Field", "Value"}, [][]string{
{"Created", utils.TimeToUTC(h.Created)},
{"Source", h.InputSource},
{"Version", h.Version},
{"Type", h.Type.String()},
{"ContainsPayloads", strconv.FormatBool(h.ContainsPayloads)},
})
os.Exit(0) // bye bye
}

// set data source
var source string
if *flagInput != "" {
@@ -153,34 +113,19 @@ func main() {
EncoderConfig: encoder.Config{
Buffer: *flagBuffer,
Compression: *flagCompress,
CSV: *flagCSV,
CSV: false,
IncludeEncoders: *flagInclude,
ExcludeEncoders: *flagExclude,
Out: *flagOutDir,
Source: source,
Version: netcap.Version,
IncludePayloads: *flagPayload,
Export: *flagExport,
Export: false,
},
BaseLayer: utils.GetBaseLayer(*flagBaseLayer),
DecodeOptions: utils.GetDecodeOptions(*flagDecodeOptions),
})

if *flagExport {
metrics.ServeMetricsAt(*flagMetricsAddress)
}

// set separators for sub structures in CSV
types.Begin = *flagBegin
types.End = *flagEnd
types.Separator = *flagStructSeparator

// read ncap file and print to stdout
if filepath.Ext(*flagInput) == ".ncap" || filepath.Ext(*flagInput) == ".gz" {
netcap.Dump(*flagInput, *flagSeparator, *flagTSV, *flagPrintStructured, *flagTable, *flagSelect, *flagUTC, *flagFields)
return
}

netcap.PrintLogo()

// print configuration as table
@@ -16,13 +16,8 @@ package main
import (
"flag"
"fmt"
"log"
"os/exec"
"strings"

"github.com/dreadl0ck/netcap"
"github.com/dreadl0ck/netcap/types"
"github.com/mgutz/ansi"
)

func printHeader() {
@@ -42,53 +37,3 @@ func printUsage() {
printHeader()
flag.PrintDefaults()
}

// CheckFields checks if the separator occurs inside fields of audit records
// to prevent this breaking the generated CSV file
func checkFields() {

r, err := netcap.Open(*flagInput)
if err != nil {
panic(err)
}

var (
h = r.ReadHeader()
record = netcap.InitRecord(h.Type)
numExpectedFields int
)
if p, ok := record.(types.AuditRecord); ok {
numExpectedFields = len(p.CSVHeader())
} else {
log.Fatal("netcap type does not implement the types.AuditRecord interface!")
}

// for {
// err = r.Next(record)
// if err != nil {
// fmt.Println(err)
// break
// }

// if p, ok := record.(types.AuditRecord); ok {
// fields := p.CSVRecord()
// // TODO refactor to use netcap lib to read file instead of calling it as command
// }
// }

r.Close()

// call netcap and parse output line by line
out, err := exec.Command("netcap", "-r", *flagInput).Output()
if err != nil {
panic(err)
}

// iterate over lines
for _, line := range strings.Split(string(out), "\n") {
count := strings.Count(line, *flagSeparator)
if count != numExpectedFields-1 {
fmt.Println(strings.Replace(line, *flagSeparator, ansi.Red+*flagSeparator+ansi.Reset, -1), ansi.Red, count, ansi.Reset)
}
}
}
@@ -0,0 +1,33 @@
/*
* NETCAP - Traffic Analysis Framework
* Copyright (c) 2017 Philipp Mieden <dreadl0ck [at] protonmail [dot] ch>
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

package main

import "flag"

var (
// dump
flagSelect = flag.String("select", "", "select specific fields of an audit records when generating csv or tables")
flagFields = flag.Bool("fields", false, "print available fields for an audit record file and exit")
flagSeparator = flag.String("sep", ",", "set separator string for csv output")
flagCSV = flag.Bool("csv", false, "print output data as csv with header line")
flagPrintStructured = flag.Bool("struc", false, "print output as structured objects")
flagTSV = flag.Bool("tsv", false, "print output as tab separated values")
flagHeader = flag.Bool("header", false, "print audit record file header and exit")
flagTable = flag.Bool("table", false, "print output as table view (thanks @evilsocket)")
flagBegin = flag.String("begin", "(", "begin character for a structure in CSV output")
flagEnd = flag.String("end", ")", "end character for a structure in CSV output")
flagStructSeparator = flag.String("struct-sep", "-", "separator character for a structure in CSV output")
flagUTC = flag.Bool("utc", false, "print timestamps as UTC when using select csv")
flagInput = flag.String("r", "", "read specified file, can either be a pcap or netcap audit record file")
)
@@ -1,3 +1,66 @@
/*
* NETCAP - Traffic Analysis Framework
* Copyright (c) 2017 Philipp Mieden <dreadl0ck [at] protonmail [dot] ch>
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

package main

func main() {}
import (
"flag"
"os"
"path/filepath"
"strconv"

"github.com/dreadl0ck/netcap"
"github.com/dreadl0ck/netcap/types"
"github.com/dreadl0ck/netcap/utils"
"github.com/evilsocket/islazy/tui"
)

func main() {

flag.Parse()

// read dumpfile header and exit
if *flagHeader {

// open input file for reading
r, err := netcap.Open(*flagInput)
if err != nil {
panic(err)
}

// get header
// this will panic if the header is corrupted
h := r.ReadHeader()

// print result as table
tui.Table(os.Stdout, []string{"Field", "Value"}, [][]string{
{"Created", utils.TimeToUTC(h.Created)},
{"Source", h.InputSource},
{"Version", h.Version},
{"Type", h.Type.String()},
{"ContainsPayloads", strconv.FormatBool(h.ContainsPayloads)},
})
os.Exit(0) // bye bye
}

// set separators for sub structures in CSV
types.Begin = *flagBegin
types.End = *flagEnd
types.Separator = *flagStructSeparator

// read ncap file and print to stdout
if filepath.Ext(*flagInput) == ".ncap" || filepath.Ext(*flagInput) == ".gz" {
netcap.Dump(*flagInput, *flagSeparator, *flagTSV, *flagPrintStructured, *flagTable, *flagSelect, *flagUTC, *flagFields)
return
}
}
@@ -1,3 +1,16 @@
/*
* NETCAP - Traffic Analysis Framework
* Copyright (c) 2017 Philipp Mieden <dreadl0ck [at] protonmail [dot] ch>
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

package main

import "flag"
@@ -8,4 +21,26 @@ var (
flagQuiet = flag.Bool("quiet", false, "dont print logo only output")
flagRead = flag.String("r", "", "input netcap audit recod file")
flagReplay = flag.Bool("replay", true, "replay traffic")

flagExport = flag.Bool("export", true, "export prometheus metrics")
flagMetricsAddress = flag.String("address", "127.0.0.1:7777", "metrics address")

flagInput = flag.String("r", "", "read specified file, can either be a pcap or netcap audit record file")
flagInterface = flag.String("iface", "", "attach to network interface and capture in live mode")
flagWorkers = flag.Int("workers", 1000, "number of workers")
flagPacketBuffer = flag.Int("pbuf", 100, "set packet buffer size, for channels that feed data to workers")
flagIngoreUnknown = flag.Bool("ignore-unknown", false, "disable writing unknown packets into a pcap file")
flagPromiscMode = flag.Bool("promisc", true, "toggle promiscous mode for live capture")
flagSnapLen = flag.Int("snaplen", 1024, "configure snaplen for live capture from interface")

flagBaseLayer = flag.String("base", "ethernet", "select base layer")
flagDecodeOptions = flag.String("opts", "lazy", "select decoding options")
flagPayload = flag.Bool("payload", false, "capture payload for supported layers")
flagCompress = flag.Bool("comp", true, "compress output with gzip")
flagBuffer = flag.Bool("buf", true, "buffer data in memory before writing to disk")
flagOutDir = flag.String("out", "", "specify output directory, will be created if it does not exist")
flagBPF = flag.String("bpf", "", "supply a BPF filter to use prior to processing packets with netcap")
flagInclude = flag.String("include", "", "include specific encoders")
flagExclude = flag.String("exclude", "", "exclude specific encoders")
flagMemProfile = flag.Bool("memprof", false, "create memory profile")
)

0 comments on commit e777c51

Please sign in to comment.
You can’t perform that action at this time.