From 2301c7296a12178bde01c65b55870a54d5e59ff2 Mon Sep 17 00:00:00 2001 From: TacoVox Date: Fri, 4 May 2018 10:19:14 +0200 Subject: [PATCH 1/3] Add Support for LCM Defragmentation This patch includes support for defragmentation of LCM packages, as specified in the LCM protocol description (https://lcm-proj.github.io/udp_multicast_protocol.html). This patch code should also be used in combination with an upcoming PR towards the GitHub LCM repository that adds golang + gopacket support. --- .travis.golint.sh | 2 +- .travis.govet.sh | 2 +- defrag/lcmdefrag.go | 143 +++++++++++++++++++++++++++++++++++++++ defrag/lcmdefrag_test.go | 105 ++++++++++++++++++++++++++++ gc | 6 ++ 5 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 defrag/lcmdefrag.go create mode 100644 defrag/lcmdefrag_test.go diff --git a/.travis.golint.sh b/.travis.golint.sh index edc64596b..763e14236 100755 --- a/.travis.golint.sh +++ b/.travis.golint.sh @@ -3,7 +3,7 @@ cd "$(dirname $0)" go get github.com/golang/lint/golint -DIRS=". tcpassembly tcpassembly/tcpreader ip4defrag reassembly macs pcapgo pcap afpacket pfring routing" +DIRS=". tcpassembly tcpassembly/tcpreader ip4defrag reassembly macs pcapgo pcap afpacket pfring routing defrag" # Add subdirectories here as we clean up golint on each. for subdir in $DIRS; do pushd $subdir diff --git a/.travis.govet.sh b/.travis.govet.sh index 52ad08413..7c2e4541b 100755 --- a/.travis.govet.sh +++ b/.travis.govet.sh @@ -1,7 +1,7 @@ #!/bin/bash cd "$(dirname $0)" -DIRS=". layers pcap pcapgo pfring tcpassembly tcpassembly/tcpreader routing ip4defrag bytediff macs" +DIRS=". layers pcap pcapgo pfring tcpassembly tcpassembly/tcpreader routing ip4defrag bytediff macs defrag" set -e for subdir in $DIRS; do pushd $subdir diff --git a/defrag/lcmdefrag.go b/defrag/lcmdefrag.go new file mode 100644 index 000000000..205adff1e --- /dev/null +++ b/defrag/lcmdefrag.go @@ -0,0 +1,143 @@ +// Copyright 2018 Google, Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. + +package defrag + +import ( + "fmt" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" +) + +const ( + // Packages are cleaned up/removed after no input was received for this + // amount of seconds. + timeout time.Duration = 3 * time.Second +) + +type lcmPacket struct { + lastPacket time.Time + done bool + recFrags uint16 + totalFrags uint16 + frags map[uint16]*layers.LCM +} + +// LCMDefragmenter supports defragmentation of LCM messages. +// +// References +// https://lcm-proj.github.io/ +// https://github.com/lcm-proj/lcm +type LCMDefragmenter struct { + packets map[uint32]*lcmPacket +} + +func newLCMPacket(totalFrags uint16) *lcmPacket { + return &lcmPacket{ + done: false, + recFrags: 0, + totalFrags: totalFrags, + frags: make(map[uint16]*layers.LCM), + } +} + +// NewLCMDefragmenter returns a new LCMDefragmenter. +func NewLCMDefragmenter() *LCMDefragmenter { + return &LCMDefragmenter{ + packets: make(map[uint32]*lcmPacket), + } +} + +func (lp *lcmPacket) append(in *layers.LCM) { + lp.frags[in.FragmentNumber] = in + lp.recFrags++ + lp.lastPacket = time.Now() +} + +func (lp *lcmPacket) assemble() (out *layers.LCM, err error) { + var blob []byte + + //Extract packets + for i := uint16(0); i < lp.totalFrags; i++ { + fragment, ok := lp.frags[i] + if !ok { + err = fmt.Errorf("Tried to defragment incomplete packet. Waiting "+ + "for more potential (unordered) packets... %d", i) + return + } + + // For the very first packet, we also want the header. + if i == 0 { + blob = append(blob, fragment.LayerContents()...) + } + + // Append the data for each packet. + blob = append(blob, fragment.Payload()...) + } + + packet := gopacket.NewPacket(blob, layers.LayerTypeLCM, gopacket.NoCopy) + lcmHdrLayer := packet.Layer(layers.LayerTypeLCM) + out, ok := lcmHdrLayer.(*layers.LCM) + if !ok { + err = fmt.Errorf("Error while decoding the defragmented packet. " + + "Erasing/dropping packet.") + } + + lp.done = true + + return +} + +func (ld *LCMDefragmenter) cleanUp() { + for key, packet := range ld.packets { + if packet.done || time.Now().Sub(packet.lastPacket) > timeout { + delete(ld.packets, key) + } + } +} + +// Defrag takes a reference to an LCM packet and processes it. +// In case the packet does not need to be defragmented, it immediately returns +// the as in passed reference. In case in was the last missing fragment, out +// will be the defragmented packet. If in was a fragment, but we are awaiting +// more, out will be set to nil. +// In the case that in was nil, we will just run the internal cleanup of the +// defragmenter that times out packages. +// If an error was encountered during defragmentation, out will also be nil, +// while err will contain further information on the failure. +func (ld *LCMDefragmenter) Defrag(in *layers.LCM) (out *layers.LCM, err error) { + // Timeout old packages and erase error prone ones. + ld.cleanUp() + + // For running cleanup only + if in == nil { + return + } + + // Quick check if this is acutally a single packet. In that case, just + // return it quickly. + if !in.Fragmented { + out = in + return + } + + // Do we need to start a new fragments obj? + if _, ok := ld.packets[in.SequenceNumber]; !ok { + ld.packets[in.SequenceNumber] = newLCMPacket(in.TotalFragments) + } + + // Append the packet + ld.packets[in.SequenceNumber].append(in) + + // Check if this is the last package of that series + if ld.packets[in.SequenceNumber].recFrags == in.TotalFragments { + out, err = ld.packets[in.SequenceNumber].assemble() + } + + return +} diff --git a/defrag/lcmdefrag_test.go b/defrag/lcmdefrag_test.go new file mode 100644 index 000000000..fb6180595 --- /dev/null +++ b/defrag/lcmdefrag_test.go @@ -0,0 +1,105 @@ +// Copyright 2018 Google, Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. + +package defrag + +import ( + "testing" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" +) + +var ( + fragmentOne = []byte{ + 0x4c, 0x43, 0x30, 0x33, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0d, + 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x02, 0x4c, 0x43, 0x4d, 0x5f, + 0x53, 0x45, 0x4c, 0x46, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x00, 0x6c, 0x63, + 0x6d, 0x20, 0x73, 0x65, 0x6c, 0x66, 0x20, 0x74, 0x65, 0x73, 0x74, + } + + fragmentTwo = []byte{ + 0x4c, 0x43, 0x30, 0x33, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0d, + 0x00, 0x00, 0x00, 0x2d, 0x00, 0x01, 0x00, 0x02, 0x6c, 0x63, 0x6d, 0x20, + 0x73, 0x65, 0x6c, 0x66, 0x20, 0x74, 0x65, 0x73, 0x74, + } + + completePacket = []byte{ + 0x4c, 0x43, 0x30, 0x32, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x43, 0x4d, 0x5f, + 0x53, 0x45, 0x4c, 0x46, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x00, 0x6c, 0x63, + 0x6d, 0x20, 0x73, 0x65, 0x6c, 0x66, 0x20, 0x74, 0x65, 0x73, 0x74, + } +) + +func TestOrderedLCMDefrag(t *testing.T) { + defragmenter := NewLCMDefragmenter() + var err error + + packet := gopacket.NewPacket(fragmentOne, layers.LayerTypeLCM, gopacket.NoCopy) + lcm := packet.Layer(layers.LayerTypeLCM).(*layers.LCM) + + lcm, err = defragmenter.Defrag(lcm) + if lcm != nil { + t.Fatal("Returned incomplete LCM message.") + } + if err != nil { + t.Fatal(err) + } + + packet = gopacket.NewPacket(fragmentTwo, layers.LayerTypeLCM, gopacket.NoCopy) + lcm = packet.Layer(layers.LayerTypeLCM).(*layers.LCM) + + lcm, err = defragmenter.Defrag(lcm) + if lcm == nil { + t.Fatal("Did not receive defragmented LCM message.") + } + if err != nil { + t.Fatal(err) + } +} + +func TestUnorderedLCMDefrag(t *testing.T) { + defragmenter := NewLCMDefragmenter() + var err error + + packet := gopacket.NewPacket(fragmentTwo, layers.LayerTypeLCM, gopacket.NoCopy) + lcm := packet.Layer(layers.LayerTypeLCM).(*layers.LCM) + + lcm, err = defragmenter.Defrag(lcm) + if lcm != nil { + t.Fatal("Returned incomplete LCM message.") + } + if err != nil { + t.Fatal(err) + } + + packet = gopacket.NewPacket(fragmentOne, layers.LayerTypeLCM, gopacket.NoCopy) + lcm = packet.Layer(layers.LayerTypeLCM).(*layers.LCM) + + lcm, err = defragmenter.Defrag(lcm) + if lcm == nil { + t.Fatal("Did not receive defragmented LCM message.") + } + if err != nil { + t.Fatal(err) + } +} + +func TestNonLCMDefrag(t *testing.T) { + defragmenter := NewLCMDefragmenter() + var err error + + packet := gopacket.NewPacket(completePacket, layers.LayerTypeLCM, gopacket.NoCopy) + lcm := packet.Layer(layers.LayerTypeLCM).(*layers.LCM) + + lcm, err = defragmenter.Defrag(lcm) + if lcm == nil { + t.Fatal("Did not receive complete LCM message.") + } + if err != nil { + t.Fatal(err) + } +} diff --git a/gc b/gc index 21f5c298e..b1d8d2e1f 100755 --- a/gc +++ b/gc @@ -186,6 +186,12 @@ if [ -f /usr/include/pfring.h ]; then go build popd fi +pushd ip4defrag +go test ./... +popd +pushd defrag +go test ./... +popd for travis_script in `ls .travis.*.sh`; do ./$travis_script From 17d5b3e38dd8415f441e257e0157db4644fb2fbf Mon Sep 17 00:00:00 2001 From: TacoVox Date: Tue, 8 May 2018 07:46:41 +0200 Subject: [PATCH 2/3] Fixed NextLayerType() Return As the LCM defragmenter will build the defragmented package under the header of the very first fragment, NextLayerType() should return the matching LayerType() in case the message is not fragmented or the first message of a fragmented one. Otherwise (all other fragmented messages), the method should return LayerTypeFragment. This patch fixes that. --- layers/lcm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/layers/lcm.go b/layers/lcm.go index 5c3293b4f..effb1af7f 100644 --- a/layers/lcm.go +++ b/layers/lcm.go @@ -165,11 +165,11 @@ func (lcm LCM) CanDecode() gopacket.LayerClass { // combination of data types, lookup of correct layer type is based on that // fingerprint. func (lcm LCM) NextLayerType() gopacket.LayerType { - if lcm.Fragmented { - return gopacket.LayerTypeFragment + if !lcm.Fragmented || (lcm.Fragmented && lcm.FragmentNumber == 0) { + return GetLCMLayerType(lcm.fingerprint) } - return GetLCMLayerType(lcm.fingerprint) + return gopacket.LayerTypeFragment } // LayerType returns LayerTypeLCM From 11f31cf5e730ac24a6c599e1f4089872d9a57463 Mon Sep 17 00:00:00 2001 From: TacoVox Date: Mon, 4 Jun 2018 07:31:14 +0200 Subject: [PATCH 3/3] Moved Defragmenter to New Folder Structure As discussed in https://github.com/google/gopacket/pull/449, the defragmenter was moved to a seperate package that will only contain the LCM defragmenter. This is done in order to enable a more clean package setup. --- .travis.golint.sh | 2 +- .travis.govet.sh | 2 +- defrag/{ => lcmdefrag}/lcmdefrag.go | 3 ++- defrag/{ => lcmdefrag}/lcmdefrag_test.go | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) rename defrag/{ => lcmdefrag}/lcmdefrag.go (97%) rename defrag/{ => lcmdefrag}/lcmdefrag_test.go (99%) diff --git a/.travis.golint.sh b/.travis.golint.sh index 763e14236..9f653bc57 100755 --- a/.travis.golint.sh +++ b/.travis.golint.sh @@ -3,7 +3,7 @@ cd "$(dirname $0)" go get github.com/golang/lint/golint -DIRS=". tcpassembly tcpassembly/tcpreader ip4defrag reassembly macs pcapgo pcap afpacket pfring routing defrag" +DIRS=". tcpassembly tcpassembly/tcpreader ip4defrag reassembly macs pcapgo pcap afpacket pfring routing defrag/lcmdefrag" # Add subdirectories here as we clean up golint on each. for subdir in $DIRS; do pushd $subdir diff --git a/.travis.govet.sh b/.travis.govet.sh index 7c2e4541b..aef4e4749 100755 --- a/.travis.govet.sh +++ b/.travis.govet.sh @@ -1,7 +1,7 @@ #!/bin/bash cd "$(dirname $0)" -DIRS=". layers pcap pcapgo pfring tcpassembly tcpassembly/tcpreader routing ip4defrag bytediff macs defrag" +DIRS=". layers pcap pcapgo pfring tcpassembly tcpassembly/tcpreader routing ip4defrag bytediff macs defrag/lcmdefrag" set -e for subdir in $DIRS; do pushd $subdir diff --git a/defrag/lcmdefrag.go b/defrag/lcmdefrag/lcmdefrag.go similarity index 97% rename from defrag/lcmdefrag.go rename to defrag/lcmdefrag/lcmdefrag.go index 205adff1e..a4cf87701 100644 --- a/defrag/lcmdefrag.go +++ b/defrag/lcmdefrag/lcmdefrag.go @@ -4,7 +4,8 @@ // that can be found in the LICENSE file in the root of the source // tree. -package defrag +// Package lcmdefrag contains a defragmenter for LCM messages. +package lcmdefrag import ( "fmt" diff --git a/defrag/lcmdefrag_test.go b/defrag/lcmdefrag/lcmdefrag_test.go similarity index 99% rename from defrag/lcmdefrag_test.go rename to defrag/lcmdefrag/lcmdefrag_test.go index fb6180595..3d54191b4 100644 --- a/defrag/lcmdefrag_test.go +++ b/defrag/lcmdefrag/lcmdefrag_test.go @@ -4,7 +4,7 @@ // that can be found in the LICENSE file in the root of the source // tree. -package defrag +package lcmdefrag import ( "testing"