[pcap][capture] #34: split directions for libpcap

Do not rely on BPF, but rather on `handle.SetDirection`, which, ironically
also fails on MacOS.

Currently uses `ReadPacketData` instead of `ZeroCopyReadPacketData` due to
inconsistencies observed with goProbe's stable version.

Also removes the fako1024 gopacket fork dependencies everywhere except for
the afpacket case

addon/Makefile

@@ -106,7 +106,7 @@ go_install:
 	cp goprobe.service absolute/etc/systemd/system/goprobe.service
 
 	echo "*** generating example configuration ***"
-	echo -e "{\n\t\"db_path\" : \"/usr/local/$(GO_PRODUCT)/db\",\n\t\"interfaces\" : {\n\t\t\"eth0\" : {\n\t\t\t\"bpf_filter\" : \"not arp and not icmp\",\n\t\t\t\"buf_size\" : 2097152,\n\t\t\t\"promisc\" : false\n\t\t}\n\t}\n}" > absolute/etc/goprobe.conf.example
+	echo -e '{"db_path":"/usr/local/db","interfaces":{"eth0":{"bpf_filter":"not arp and not icmp","buf_size":2097152,"promisc":false}},"logging":{"destination":"console","level":"debug"}}' > absolute/etc/goprobe.conf.example
 
 	#set the appropriate permissions for files
 	chmod -R 755 absolute/bin

cmd/goProbe/goProbe.go

@@ -265,7 +265,7 @@ func handleRotations(manager *capture.Manager, logger log.Logger) {
 
 			manager.LastRotation = time.Now()
 			woChan := make(chan capture.TaggedAggFlowMap, capture.MaxIfaces)
-			writeoutsChan <- capture.Writeout{woChan, captureManager.LastRotation}
+			writeoutsChan <- capture.Writeout{Chan: woChan, Timestamp: captureManager.LastRotation}
 			manager.RotateAll(woChan)
 			close(woChan)
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/fako1024/gopacket v1.1.20-0.20220816131758-4518db4bf798
 	github.com/go-chi/chi/v5 v5.0.7
 	github.com/go-chi/docgen v1.2.0
+	github.com/google/gopacket v1.1.19
 	github.com/json-iterator/go v1.1.12
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0

go.sum

@@ -19,6 +19,8 @@ github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -60,10 +62,16 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
 golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -74,6 +82,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME=
 golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/capture/GPLog.go

@@ -18,9 +18,9 @@ import (
 	"os"
 	"sync"
 
-	"github.com/fako1024/gopacket"
-	"github.com/fako1024/gopacket/layers"
-	"github.com/fako1024/gopacket/pcapgo"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcapgo"
 )
 
 // PacketLogWriter provides methods to write packets to file for a given interface

pkg/capture/GPPacket.go

@@ -17,8 +17,8 @@ package capture
 import (
 	"fmt"
 
-	"github.com/fako1024/gopacket"
-	"github.com/fako1024/gopacket/layers"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
 )
 
 var (
@@ -92,7 +92,7 @@ func (p *GPPacket) computeEPHash() {
 }
 
 // Populate takes a raw packet and populates a GPPacket structure from it.
-func (p *GPPacket) Populate(srcPacket gopacket.Packet) error {
+func (p *GPPacket) Populate(srcPacket gopacket.Packet, inbound bool) error {
 
 	// first things first: reset packet from previous run
 	p.reset()
@@ -104,10 +104,7 @@ func (p *GPPacket) Populate(srcPacket gopacket.Packet) error {
 	p.numBytes = uint16(srcPacket.Metadata().CaptureInfo.Length)
 
 	// read the direction from which the packet entered the interface
-	p.dirInbound = false
-	if srcPacket.Metadata().CaptureInfo.Inbound == 1 {
-		p.dirInbound = true
-	}
+	p.dirInbound = inbound
 
 	// for ESP traffic (which lacks a transport layer)
 	var skipTransport bool

pkg/capture/afpacket_source_linux.go

@@ -6,31 +6,39 @@ import (
 	"strings"
 	"time"
 
-	"github.com/fako1024/gopacket"
 	"github.com/fako1024/gopacket/afpacket"
-	"github.com/fako1024/gopacket/layers"
-	"github.com/fako1024/gopacket/pcap"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
 	"golang.org/x/net/bpf"
 )
 
 type AFPacketSource struct {
 	handle *afpacket.TPacket
 }
 
-func (p *AFPacketSource) NextPacket() (gopacket.Packet, error) {
+func (p *AFPacketSource) NextPacket() (Packet, error) {
 	data, ci, err := p.handle.ZeroCopyReadPacketData()
 	if err != nil {
-		return nil, err
+		return Packet{}, err
+	}
+
+	// translates the capture info from fako's gopacket to the standard google one. An unfortunate
+	// detour, but necessary as long as the afpacket additions aren't part of the google upstream
+	gpci := gopacket.CaptureInfo{
+		AncillaryData:  ci.AncillaryData,
+		CaptureLength:  ci.CaptureLength,
+		InterfaceIndex: ci.InterfaceIndex,
+		Length:         ci.Length,
+		Timestamp:      ci.Timestamp,
 	}
 
 	packet := gopacket.NewPacket(data, layers.LinkTypeEthernet, defaultDecodeOptions)
 	m := packet.Metadata()
-	m.CaptureInfo = ci
+	m.CaptureInfo = gpci
 	m.Truncated = m.Truncated || ci.CaptureLength < ci.Length
 
-	// fmt.Println("GOT AF_PACKET", packet.String())
-
-	return packet, nil
+	return Packet{packet: packet, inbound: ci.Inbound == 1}, nil
 }
 
 func (p *AFPacketSource) Init(iface, bpfFilter string, captureLength, bufSize int, promisc bool) error {

pkg/capture/capture.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/els0r/goProbe/pkg/goDB"
 	"github.com/els0r/log"
-	"github.com/fako1024/gopacket/pcap"
+	"github.com/google/gopacket/pcap"
 )
 
 const (
@@ -426,9 +426,13 @@ func (c *Capture) process() {
 			return fmt.Errorf("capture error: %s", err)
 		}
 
-		if err := gppacket.Populate(packet); err == nil {
+		if err := gppacket.Populate(packet.packet, packet.inbound); err == nil {
 
-			//fmt.Println("Packet on PCAP:", goDB.RawIPToString(gppacket.sip[:]), "->", goDB.RawIPToString(gppacket.dip[:]), strconv.Itoa(int(uint16(gppacket.dport[0])<<8|uint16(gppacket.dport[1]))), gppacket.numBytes, base64.RawStdEncoding.EncodeToString(gppacket.epHash[:]), base64.RawStdEncoding.EncodeToString(gppacket.epHashReverse[:]))
+			// fmt.Println("Packet captured:", goDB.RawIPToString(gppacket.sip[:]), "->", goDB.RawIPToString(gppacket.dip[:]),
+			// 	strconv.Itoa(int(uint16(gppacket.dport[0])<<8|uint16(gppacket.dport[1]))),
+			// 	gppacket.numBytes,
+			// 	base64.RawStdEncoding.EncodeToString(gppacket.epHash[:]), base64.RawStdEncoding.EncodeToString(gppacket.epHashReverse[:]),
+			// )
 
 			c.flowLog.Add(&gppacket)
 			errcount = 0
@@ -440,7 +444,7 @@ func (c *Capture) process() {
 			// of the error would be taken, which results in a non-minimal set of errors
 			if _, exists := c.errMap[err.Error()]; !exists {
 				// log the packet to the pcap error logs
-				if logerr := PacketLog.Log(c.iface, packet, Snaplen); logerr != nil {
+				if logerr := PacketLog.Log(c.iface, packet.packet, Snaplen); logerr != nil {
 					c.logger.Info("failed to log faulty packet: " + logerr.Error())
 				}
 			}

pkg/capture/capture_default.go

@@ -3,7 +3,7 @@
 
 package capture
 
-import "github.com/fako1024/gopacket/pcap"
+import "github.com/google/gopacket/pcap"
 
 var errCaptureTimeout = pcap.NextErrorTimeoutExpired