Merge d074a39 into 6878658

README.md

@@ -14,7 +14,7 @@ docker run --network=host -v ~/data:/home/ -it measurementlab/tcp-info -prom=707
 
 # Fast tcp-info collector in Go
 
-This repository uses the netlink API to collect inet_diag messages, partially parses them, caches the intermediate representation.
+This repository uses the netlink API to collect inet_diag messages, partially parses them, and caches the intermediate representation.
 It then detects differences from one scan to the next, and queues connections that have changed for logging.
 It logs the intermediate representation through external zstd processes to one file per connection.
 
@@ -35,10 +35,6 @@ sudo apt-get update && sudo apt-get install -y zstd
 
 # Parse library and command line tools
 
-## The *parse* package
-
-This package is intended to be used as a library for command line tools and other applications that need to parse the ArchivedRecord messages.
-
 ## CSV tool
 
 The cmd/csvtool directory contains a tool for parsing ArchivedRecord and producing CSV files.  Currently reads netlink-jSONL from stdin and writes CSV to stdout.

collector/socket-monitor.go

@@ -17,8 +17,8 @@ import (
 // TODO - Figure out why we aren't seeing INET_DIAG_DCTCPINFO or INET_DIAG_BBRINFO messages.
 func makeReq(inetType uint8) *nl.NetlinkRequest {
 	req := nl.NewNetlinkRequest(inetdiag.SOCK_DIAG_BY_FAMILY, syscall.NLM_F_DUMP|syscall.NLM_F_REQUEST)
-	msg := inetdiag.NewInetDiagReqV2(inetType, syscall.IPPROTO_TCP,
-		inetdiag.TCPF_ALL & ^((1<<uint(tcp.SYN_RECV))|(1<<uint(tcp.TIME_WAIT))|(1<<uint(tcp.CLOSE))))
+	msg := inetdiag.NewReqV2(inetType, syscall.IPPROTO_TCP,
+		tcp.AllFlags & ^((1<<uint(tcp.SYN_RECV))|(1<<uint(tcp.TIME_WAIT))|(1<<uint(tcp.CLOSE))))
 	msg.IDiagExt |= (1 << (inetdiag.INET_DIAG_MEMINFO - 1))
 	msg.IDiagExt |= (1 << (inetdiag.INET_DIAG_INFO - 1))
 	msg.IDiagExt |= (1 << (inetdiag.INET_DIAG_VEGASINFO - 1))

inetdiag/inetdiag.go

@@ -59,6 +59,7 @@ const (
 	INET_DIAG_MAX
 )
 
+// InetDiagType provides human readable strings for decoding attribute types.
 var InetDiagType = map[int32]string{
 	INET_DIAG_MEMINFO:   "MemInfo",
 	INET_DIAG_INFO:      "TCPInfo",
@@ -85,27 +86,24 @@ var diagFamilyMap = map[uint8]string{
 	syscall.AF_INET6: "tcp6",
 }
 
-//	if (tb[INET_DIAG_PROTOCOL])
-//		s->raw_prot = rta_getattr_u8(tb[INET_DIAG_PROTOCOL]);
+// Protocol defines the type corresponding to INET_DIAG_PROTOCOL 8 bit field.
 type Protocol uint8
 
 const (
+	// Protocol_IPPROTO_UNUSED ...
 	Protocol_IPPROTO_UNUSED Protocol = 0
-	Protocol_IPPROTO_TCP    Protocol = 6
-	Protocol_IPPROTO_UDP    Protocol = 17
-	Protocol_IPPROTO_DCCP   Protocol = 33
+	// Protocol_IPPROTO_TCP    indicates TCP traffic.
+	Protocol_IPPROTO_TCP Protocol = 6
+	// Protocol_IPPROTO_UDP   indicates UDP traffic.
+	Protocol_IPPROTO_UDP Protocol = 17
+	// Protocol_IPPROTO_DCCP indicates DCCP traffic.
+	Protocol_IPPROTO_DCCP Protocol = 33
 )
 
-var Protocol_name = map[int32]string{
+// ProtocolName is used to convert Protocol values to strings.
+var ProtocolName = map[int32]string{
 	0:  "IPPROTO_UNUSED",
 	6:  "IPPROTO_TCP",
 	17: "IPPROTO_UDP",
 	33: "IPPROTO_DCCP",
 }
-
-var Protocol_value = map[string]int32{
-	"IPPROTO_UNUSED": 0,
-	"IPPROTO_TCP":    6,
-	"IPPROTO_UDP":    17,
-	"IPPROTO_DCCP":   33,
-}

inetdiag/inetdiag_test.go

@@ -9,18 +9,18 @@ import (
 )
 
 func TestSizes(t *testing.T) {
-	if unsafe.Sizeof(inetdiag.InetDiagSockID{}) != 48 {
-		t.Error("SockID wrong size", unsafe.Sizeof(inetdiag.InetDiagSockID{}))
+	if unsafe.Sizeof(inetdiag.SockID{}) != 48 {
+		t.Error("SockID wrong size", unsafe.Sizeof(inetdiag.SockID{}))
 	}
 
 	hdr := inetdiag.InetDiagMsg{}
-	if unsafe.Sizeof(hdr) != 4*6+unsafe.Sizeof(inetdiag.InetDiagSockID{}) {
+	if unsafe.Sizeof(hdr) != 4*6+unsafe.Sizeof(inetdiag.SockID{}) {
 		t.Error("Header is wrong size", unsafe.Sizeof(hdr))
 	}
 }
 
 func TestInetDiagReqV2Serialize(t *testing.T) {
-	v2 := inetdiag.NewInetDiagReqV2(syscall.AF_INET, 23, 0x0E)
+	v2 := inetdiag.NewReqV2(syscall.AF_INET, 23, 0x0E)
 	data := v2.Serialize()
 	if v2.Len() != len(data) {
 		t.Error(data, "should be length", v2.Len())

inetdiag/structs.go

@@ -36,8 +36,6 @@ const (
 	SOCK_DIAG_BY_FAMILY = 20 // uapi/linux/sock_diag.h
 )
 
-const TCPF_ALL = 0xFFF
-
 var (
 	// ErrBadPid is used when the PID is mismatched between the netlink socket and the calling process.
 	ErrBadPid = errors.New("bad PID, can't listen to NL socket")
@@ -49,44 +47,44 @@ var (
 	ErrBadMsgData = errors.New("bad message data from netlink message")
 )
 
-// InetDiagReqV2 is the Netlink request struct, as in linux/inet_diag.h
+// ReqV2 is the Netlink request struct, as in linux/inet_diag.h
 // Note that netlink messages use host byte ordering, unless NLA_F_NET_BYTEORDER flag is present.
-type InetDiagReqV2 struct {
+type ReqV2 struct {
 	SDiagFamily   uint8
 	SDiagProtocol uint8
 	IDiagExt      uint8
 	Pad           uint8
 	IDiagStates   uint32
-	ID            InetDiagSockID
+	ID            SockID
 }
 
-// SizeofInetDiagReqV2 is the size of the struct.
+// SizeofReqV2 is the size of the struct.
 // TODO should we just make this explicit in the code?
-const SizeofInetDiagReqV2 = int(unsafe.Sizeof(InetDiagReqV2{})) // Should be 0x38
+const SizeofReqV2 = int(unsafe.Sizeof(ReqV2{})) // Should be 0x38
 
 // Serialize is provided for json serialization?
 // TODO - should use binary functions instead?
-func (req *InetDiagReqV2) Serialize() []byte {
-	return (*(*[SizeofInetDiagReqV2]byte)(unsafe.Pointer(req)))[:]
+func (req *ReqV2) Serialize() []byte {
+	return (*(*[SizeofReqV2]byte)(unsafe.Pointer(req)))[:]
 }
 
 // Len is provided for json serialization?
-func (req *InetDiagReqV2) Len() int {
-	return SizeofInetDiagReqV2
+func (req *ReqV2) Len() int {
+	return SizeofReqV2
 }
 
-// NewInetDiagReqV2 creates a new request.
-func NewInetDiagReqV2(family, protocol uint8, states uint32) *InetDiagReqV2 {
-	return &InetDiagReqV2{
+// NewReqV2 creates a new request.
+func NewReqV2(family, protocol uint8, states uint32) *ReqV2 {
+	return &ReqV2{
 		SDiagFamily:   family,
 		SDiagProtocol: protocol,
 		IDiagStates:   states,
 	}
 }
 
-// InetDiagSockID is the binary linux representation of a socket, as in linux/inet_diag.h
+// SockID is the binary linux representation of a socket, as in linux/inet_diag.h
 // Linux code comments indicate this struct uses the network byte order!!!
-type InetDiagSockID struct {
+type SockID struct {
 	IDiagSPort [2]byte
 	IDiagDPort [2]byte
 	IDiagSrc   [16]byte
@@ -97,36 +95,36 @@ type InetDiagSockID struct {
 }
 
 // Interface returns the interface number.
-func (id *InetDiagSockID) Interface() uint32 {
+func (id *SockID) Interface() uint32 {
 	return binary.BigEndian.Uint32(id.IDiagIf[:])
 }
 
 // SrcIP returns a golang net encoding of source address.
-func (id *InetDiagSockID) SrcIP() net.IP {
+func (id *SockID) SrcIP() net.IP {
 	return ip(id.IDiagSrc)
 }
 
 // DstIP returns a golang net encoding of destination address.
-func (id *InetDiagSockID) DstIP() net.IP {
+func (id *SockID) DstIP() net.IP {
 	return ip(id.IDiagDst)
 }
 
 // SPort returns the host byte ordered port.
 // In general, Netlink is supposed to use host byte order, but this seems to be an exception.
 // Perhaps Netlink is reading a tcp stack structure that holds the port in network byte order.
-func (id *InetDiagSockID) SPort() uint16 {
+func (id *SockID) SPort() uint16 {
 	return binary.BigEndian.Uint16(id.IDiagSPort[:])
 }
 
 // DPort returns the host byte ordered port.
 // In general, Netlink is supposed to use host byte order, but this seems to be an exception.
 // Perhaps Netlink is reading a tcp stack structure that holds the port in network byte order.
-func (id *InetDiagSockID) DPort() uint16 {
+func (id *SockID) DPort() uint16 {
 	return binary.BigEndian.Uint16(id.IDiagDPort[:])
 }
 
 // Cookie returns the SockID's 64 bit unsigned cookie.
-func (id *InetDiagSockID) Cookie() uint64 {
+func (id *SockID) Cookie() uint64 {
 	// This is a socket UUID generated within the kernel, and is therefore in host byte order.
 	return binary.LittleEndian.Uint64(id.IDiagCookie[:])
 }
@@ -156,37 +154,38 @@ func ipv6(original [16]byte) net.IP {
 	return original[:]
 }
 
-// These are related to filters.  We don't currently use filters, so we ignore this type.
+// HostCond is related to filters.  We don't currently use filters, so we don't actually use this type.
 type HostCond struct { // inet_diag_hostcond
 	Family    uint8  // __u8 family
 	PrefixLen uint8  // __u8 prefix_len
 	Port      uint16 // int port
 	Addr      uint32 // __be32	addr[0];
 }
+
+// MarkCond is related to filters.  We don't currently use filters, so we don't actually use this type.
 type MarkCond struct { // inet_diag_markcond
 	Mark uint32
 	Mask uint32
 }
 
 // InetDiagMsg is the linux binary representation of a InetDiag message header, as in linux/inet_diag.h
 // Note that netlink messages use host byte ordering, unless NLA_F_NET_BYTEORDER flag is present.
-// INET_DIAG_INFO
 type InetDiagMsg struct {
 	IDiagFamily  uint8
 	IDiagState   uint8
 	IDiagTimer   uint8
 	IDiagRetrans uint8
-	ID           InetDiagSockID
+	ID           SockID
 	IDiagExpires uint32
 	IDiagRqueue  uint32
 	IDiagWqueue  uint32
 	IDiagUID     uint32
 	IDiagInode   uint32
 }
 
+// SocketMemInfo implements the struct associated with INET_DIAG_SKMEMINFO
 // Haven't found a corresponding linux struct, but the message is described
 // in https://manpages.debian.org/stretch/manpages/sock_diag.7.en.html
-// INET_DIAG_SKMEMINFO
 type SocketMemInfo struct {
 	RmemAlloc  uint32
 	Rcvbuf     uint32
@@ -199,24 +198,26 @@ type SocketMemInfo struct {
 	Drops      uint32
 }
 
-// MemInfo corresponds to the linux struct inet_diag_meminfo.
-// In is used to decode attribute Type INET_DIAG_MEMINFO
+// MemInfo implements the struct associated with INET_DIAG_MEMINFO, corresponding with
+// linux struct inet_diag_meminfo in uapi/linux/inet_diag.h.
 type MemInfo struct {
 	Rmem uint32
 	Wmem uint32
 	Fmem uint32
 	Tmem uint32
 }
 
-// INET_DIAG_VEGASINFO
+// VegasInfo implements the struct associated with INET_DIAG_VEGASINFO, corresponding with
+// linux struct tcpvegas_info in uapi/linux/inet_diag.h.
 type VegasInfo struct {
 	Enabled  uint32
 	RTTCount uint32
 	RTT      uint32
 	MinRTT   uint32
 }
 
-// INET_DIAG_DCTCPINFO
+// DCTCPInfo implements the struct associated with INET_DIAG_DCTCPINFO attribute, corresponding with
+// linux struct tcp_dctcp_info in uapi/linux/inet_diag.h.
 type DCTCPInfo struct {
 	Enabled uint16
 	CEState uint16
@@ -225,8 +226,8 @@ type DCTCPInfo struct {
 	ABTot   uint32
 }
 
-// BBRInfo corresponds to linux struct tcp_bbr_info.
-// Used for decoding attribute Type:INET_DIAG_BBRINFO
+// BBRInfo implements the struct associated with INET_DIAG_BBRINFO attribute, corresponding with
+// linux struct tcp_bbr_info in uapi/linux/inet_diag.h.
 type BBRInfo struct {
 	Bw         int64  // Max-filtered BW (app throughput) estimate in bytes/second
 	MinRtt     uint32 // Min-filtered RTT in uSec
@@ -273,22 +274,3 @@ if (tb[INET_DIAG_PEERS]) {
 		printf(",%s", format_host_sa(sa));
 }
 */
-
-/*
-
-//	INET_DIAG_SKV6ONLY
-			v6only = rta_getattr_u8(tb[INET_DIAG_SKV6ONLY]);
-
-//	INET_DIAG_SHUTDOWN
-			mask = rta_getattr_u8(tb[INET_DIAG_SHUTDOWN]);
-
-
-
-
-*/
-
-//	INET_DIAG_TOS
-//	INET_DIAG_TCLASS
-//	INET_DIAG_PAD
-//	INET_DIAG_CLASS_ID
-//	INET_DIAG_MD5SIG

netlink/netlink_test.go

@@ -30,7 +30,7 @@ func init() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 }
 
-func toString(id inetdiag.InetDiagSockID) string {
+func toString(id inetdiag.SockID) string {
 	return fmt.Sprintf("%s:%d -> %s:%d", id.SrcIP().String(), id.SPort(), id.DstIP().String(), id.DPort())
 }