Merge 30da96e into 601890b

.travis.yml

@@ -1,17 +1,15 @@
 # Travis configuration for tcp-info fast sidestream tool.
 language: go
-# We must specify go 1.12 when using logx.
 go:
- - 1.12
+ - 1.18
 
-dist: xenial
 services:
 - docker
 
 ###########################################################################
 before_install:
 # Coverage tools
-- go get github.com/mattn/goveralls
+- go install github.com/mattn/goveralls@latest
 - echo Branch is ${TRAVIS_BRANCH} and Tag is $TRAVIS_TAG
 
 install:

Dockerfile

@@ -1,5 +1,5 @@
 # An image for building zstd
-FROM ubuntu as zstd-builder
+FROM ubuntu:20.04 as zstd-builder
 
 # Get zstd source and compile zstd as a static binary.
 RUN apt-get update && apt-get update -y && apt-get install -y make gcc libc-dev git
@@ -8,7 +8,7 @@ RUN mkdir /pkg && cd /src && make MOREFLAGS="-static" zstd && make DESTDIR=/pkg
 
 
 # An image for building tcp-info
-FROM golang:1.12 as tcp-info-builder
+FROM golang:1.18 as tcp-info-builder
 
 ENV CGO_ENABLED 0
 
@@ -17,12 +17,13 @@ ADD . /go/src/github.com/m-lab/tcp-info
 WORKDIR /go/src/github.com/m-lab/tcp-info
 
 # Get all of our imports and compile the tcp-info binary into /go/bin
-RUN go get -v \
+RUN go get -v . && \
+    go install -v \
       -ldflags "-X github.com/m-lab/go/prometheusx.GitShortCommit=$(git log -1 --format=%h)" \
       .
 
 # Build the image containing both binaries.
-FROM alpine
+FROM alpine:3.16
 
 # Copy the zstd binary and license.
 COPY --from=zstd-builder /pkg/usr/local/bin/zstd /bin/zstd

Dockerfile.example

@@ -0,0 +1,13 @@
+# Build uuid-annotator
+FROM golang:1.18-alpine as build
+RUN apk --no-cache add git
+COPY . /go/src/github.com/m-lab/tcp-info
+WORKDIR /go/src/github.com/m-lab/tcp-info
+RUN go get -v ./cmd/example-eventsocket-client && \
+    go install ./cmd/example-eventsocket-client
+
+# Put it in its own image.
+FROM alpine:3.16
+COPY --from=build /go/bin/example-eventsocket-client /example-eventsocket-client
+WORKDIR /
+ENTRYPOINT ["/example-eventsocket-client"]

README.md

@@ -12,7 +12,7 @@ metrics published on port 7070:
 docker run --network=host -v ~/data:/home/ -it measurementlab/tcp-info -prom=7070
 ```
 
-# Fast tcp-info collector in Go
+## Fast tcp-info collector in Go
 
 This repository uses the netlink API to collect inet_diag messages, partially parses them, and caches the intermediate representation.
 It then detects differences from one scan to the next, and queues connections that have changed for logging.
@@ -33,14 +33,32 @@ OR
 sudo apt-get update && sudo apt-get install -y zstd
 ```
 
-# Parse library and command line tools
+## Example sidecar
 
-## CSV tool
+The tcp-info eventsocket interface allows sidecar services to receive "open" and
+"close" events on a unix domain socket connection. A simple reference
+implementation `cmd/example-eventsocket-client` can be started using
+`docker-compose`.
 
-The cmd/csvtool directory contains a tool for parsing ArchivedRecord and producing CSV files.  Currently reads netlink-jSONL from stdin and writes CSV to stdout.
+```bash
+docker-compse up
+```
+
+New TCP events are processed by the `example-eventsocket-client` sidecar and
+logged to stderr. You may trigger a TCP connection from within the TCPINFO
+container using a command like:
+
+```bash
+docker exec -it tcp-info_tcpinfo_1 wget www.google.com
+```
+
+## Parse library and command line tools
 
+### CSV tool
+
+The cmd/csvtool directory contains a tool for parsing ArchivedRecord and producing CSV files.  Currently reads netlink-jSONL from stdin and writes CSV to stdout.
 
-# Code Layout
+## Code Layout
 
 * inetdiag - code related to include/uapi/linux/inet_diag.h.  All structs will be in structs.go
 * tcp - Should include ONLY the code related to include/uapi/linux/tcp.h
@@ -50,7 +68,7 @@ The cmd/csvtool directory contains a tool for parsing ArchivedRecord and produci
 * cache - code to cache netlink messages and detect changes.
 * collector - code related to collecting netlink messages from the kernel.
 
-## Dependencies (as of March 2019)
+### Dependencies (as of March 2019)
 
 * saver: inetdiag, cache, parse, tcp, zstd
 * collector: parse, saver, inetdiag, tcp
@@ -60,14 +78,14 @@ The cmd/csvtool directory contains a tool for parsing ArchivedRecord and produci
 
 And (almost) all package use metrics.
 
-### Layers for main.go (each layer depends only on items to right, or lower layers)
+#### Layers for main.go (each layer depends only on items to right, or lower layers)
 
 1. main.go
 1. collector > saver > cache
 1. netlink > inetdiag
 1. tcp, zstd, metrics
 
-### Layers for parse package:
+#### Layers for parse package
 
 1. parse (used by command line tools, etl)
 1. netlink > inetdiag

cmd/example-eventsocket-client/main.go

@@ -0,0 +1,87 @@
+// example-eventsocket-client is a minimal reference implementation of a tcpinfo
+// eventsocket client.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"time"
+
+	"github.com/m-lab/go/flagx"
+	"github.com/m-lab/go/rtx"
+	"github.com/m-lab/tcp-info/eventsocket"
+	"github.com/m-lab/tcp-info/inetdiag"
+)
+
+var (
+	mainCtx, mainCancel = context.WithCancel(context.Background())
+)
+
+// event contains fields for an open event.
+type event struct {
+	timestamp time.Time
+	uuid      string
+	id        *inetdiag.SockID
+}
+
+// handler implements the eventsocket.Handler interface.
+type handler struct {
+	events chan event
+}
+
+// Open is called by tcp-info synchronously for every TCP open event.
+func (h *handler) Open(ctx context.Context, timestamp time.Time, uuid string, id *inetdiag.SockID) {
+	// NOTE: Until this function returns, tcp-info cannot send additional
+	// events. So, immediately attempt to send the event on a channel that will
+	// be read by an asynchronous processing goroutine.
+	select {
+	case h.events <- event{timestamp: timestamp, uuid: uuid, id: id}:
+		log.Println("open ", "sent", uuid, timestamp, id)
+	default:
+		// If the write to the events channel would have blocked, discard this
+		// event instead.
+		log.Println("open ", "skipped", uuid, timestamp, id)
+	}
+}
+
+// Close is called by tcp-info synchronously for every TCP close event.
+func (h *handler) Close(ctx context.Context, timestamp time.Time, uuid string) {
+	log.Println("close", uuid, timestamp)
+}
+
+// ProcessOpenEvents reads and processes events received by the open handler.
+func (h *handler) ProcessOpenEvents(ctx context.Context) {
+	for {
+		select {
+		case e := <-h.events:
+			log.Println("processing", e)
+		case <-ctx.Done():
+			log.Println("shutdown")
+			return
+		}
+	}
+}
+
+func main() {
+	defer mainCancel()
+
+	flag.Parse()
+	rtx.Must(flagx.ArgsFromEnv(flag.CommandLine), "could not get args from environment variables")
+
+	if *eventsocket.Filename == "" {
+		log.Fatal("-tcpinfo.eventsocket path is required")
+	}
+
+	h := &handler{events: make(chan event)}
+
+	// Process events received by the eventsocket handler. The goroutine will
+	// block until an open event occurs or the context is cancelled.
+	go h.ProcessOpenEvents(mainCtx)
+
+	// Begin listening on the eventsocket for new events, and dispatch them to
+	// the given handler.
+	go eventsocket.MustRun(mainCtx, *eventsocket.Filename, h)
+
+	<-mainCtx.Done()
+}

docker-compose.yaml

@@ -0,0 +1,41 @@
+version: '3.7'
+services:
+  tcpinfo:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./local:/local
+    # NOTE: All service containers will use the same network and IP. All ports
+    # must be configured on the first.
+    ports:
+      - target: 9990
+        published: 9990
+        protocol: tcp
+        mode: bridge
+      - target: 9991
+        published: 9991
+        protocol: tcp
+        mode: bridge
+    command:
+      - -prometheusx.listen-address=:9990
+      - -output=/local/tcpinfo
+      - -tcpinfo.eventsocket=/local/tcpevents.sock
+      - -anonymize.ip=none
+
+  example-eventsocket-client:
+    build:
+      context: .
+      dockerfile: Dockerfile.example
+    volumes:
+      - ./local:/local
+    network_mode: "service:tcpinfo"
+    deploy:
+      # NOTE: container may fail on startup if it tries to read a
+      # socket that the above services are not yet listening on.
+      # So, allow restart.
+      restart_policy:
+        condition: any
+        delay: 5s
+    command:
+      - -tcpinfo.eventsocket=/local/tcpevents.sock

go.mod

@@ -0,0 +1,26 @@
+module github.com/m-lab/tcp-info
+
+go 1.18
+
+require (
+	github.com/go-test/deep v1.0.6
+	github.com/gocarina/gocsv v0.0.0-20200827134620-49f5c3fa2b3e
+	github.com/m-lab/go v0.1.47
+	github.com/m-lab/uuid v0.0.0-20191115203855-549727171666
+	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_model v0.2.0
+	github.com/vishvananda/netlink v1.1.0
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
+)
+
+require (
+	github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.1.3 // indirect
+	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
+	google.golang.org/protobuf v1.23.0 // indirect
+)