gopacket sniffer

Makefile

@@ -26,7 +26,7 @@ $(APP_EXE): app/*.go render/*.go report/*.go xfer/*.go
 $(PROBE_EXE): probe/*.go probe/docker/*.go probe/endpoint/*.go probe/host/*.go probe/process/*.go probe/overlay/*.go report/*.go xfer/*.go
 
 $(APP_EXE) $(PROBE_EXE):
-	go get -tags netgo ./$(@D)
+	go get -d -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\" -X main.version $(SCOPE_VERSION)" -tags netgo -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	        rm $@; \

app/api_topology.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gorilla/websocket"
 
 	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/report"
 )
 
 const (
@@ -27,7 +28,7 @@ type APINode struct {
 
 // APIEdge is returned by the /api/topology/*/*/* handlers.
 type APIEdge struct {
-	Metadata render.AggregateMetadata `json:"metadata"`
+	Metadata report.EdgeMetadata `json:"metadata"`
 }
 
 // Full topology.
@@ -76,7 +77,7 @@ func handleEdge(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Req
 		localID  = vars["local"]
 		remoteID = vars["remote"]
 		rpt      = rep.Report()
-		metadata = t.renderer.AggregateMetadata(rpt, localID, remoteID)
+		metadata = t.renderer.EdgeMetadata(rpt, localID, remoteID)
 	)
 
 	respondWith(w, http.StatusOK, APIEdge{Metadata: metadata})

app/api_topology_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/render/expected"
+	"github.com/weaveworks/scope/report"
 	"github.com/weaveworks/scope/test"
 )
 
@@ -71,6 +72,7 @@ func TestAPITopologyApplications(t *testing.T) {
 		if err := json.Unmarshal(body, &topo); err != nil {
 			t.Fatal(err)
 		}
+
 		if want, have := render.OnlyConnected(expected.RenderedProcesses), fixNodeMetadatas(topo.Nodes); !reflect.DeepEqual(want, have) {
 			t.Error(test.Diff(want, have))
 		}
@@ -93,9 +95,9 @@ func TestAPITopologyApplications(t *testing.T) {
 		if err := json.Unmarshal(body, &edge); err != nil {
 			t.Fatalf("JSON parse error: %s", err)
 		}
-		if want, have := (render.AggregateMetadata{
-			"egress_bytes":  10,
-			"ingress_bytes": 100,
+		if want, have := (report.EdgeMetadata{
+			EgressPacketCount: newu64(10),
+			EgressByteCount:   newu64(100),
 		}), edge.Metadata; !reflect.DeepEqual(want, have) {
 			t.Error(test.Diff(want, have))
 		}
@@ -112,7 +114,8 @@ func TestAPITopologyHosts(t *testing.T) {
 		if err := json.Unmarshal(body, &topo); err != nil {
 			t.Fatal(err)
 		}
-		if want, have := (expected.RenderedHosts), fixNodeMetadatas(topo.Nodes); !reflect.DeepEqual(want, have) {
+
+		if want, have := expected.RenderedHosts, fixNodeMetadatas(topo.Nodes); !reflect.DeepEqual(want, have) {
 			t.Error(test.Diff(want, have))
 		}
 	}
@@ -134,11 +137,10 @@ func TestAPITopologyHosts(t *testing.T) {
 		if err := json.Unmarshal(body, &edge); err != nil {
 			t.Fatalf("JSON parse error: %s", err)
 		}
-		want := render.AggregateMetadata{
-			"max_conn_count_tcp": 3,
-		}
-		if !reflect.DeepEqual(want, edge.Metadata) {
-			t.Errorf("Edge metadata error. Want %v, have %v", want, edge)
+		if want, have := (report.EdgeMetadata{
+			MaxConnCountTCP: newu64(3),
+		}), edge.Metadata; !reflect.DeepEqual(want, have) {
+			t.Error(test.Diff(want, have))
 		}
 	}
 }
@@ -176,3 +178,5 @@ func TestAPITopologyWebsocket(t *testing.T) {
 	equals(t, 0, len(d.Update))
 	equals(t, 0, len(d.Remove))
 }
+
+func newu64(value uint64) *uint64 { return &value }

app/report_lifo.go

@@ -51,9 +51,14 @@ func NewReportLIFO(r reporter, maxAge time.Duration) *ReportLIFO {
 			case req := <-l.requests:
 				// Request for the current report.
 				report := report.MakeReport()
+				oldest := time.Now()
 				for _, r := range l.reports {
+					if r.Timestamp.Before(oldest) {
+						oldest = r.Timestamp
+					}
 					report.Merge(r.Report)
 				}
+				report.Window = time.Now().Sub(oldest)
 				req <- report
 
 			case q := <-l.quit:

circle.yml

@@ -19,7 +19,7 @@ dependencies:
   cache_directories:
     - "~/docker"
   override:
-    - sudo apt-get --only-upgrade install tar
+    - sudo apt-get --only-upgrade install tar libpcap0.8-dev
     - bin/rebuild-ui-build-image
     - curl https://sdk.cloud.google.com | bash
     - test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD"
@@ -40,8 +40,7 @@ test:
     - cd $SRCDIR; ./bin/lint .
     - cd $SRCDIR; make client-test
     - cd $SRCDIR; make static
-    - cd $SRCDIR; rm -f app/app probe/probe; GOOS=darwin make && make clean
-    - cd $SRCDIR; rm -f app/app probe/probe; GOOS=linux make
+    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make
     - cd $SRCDIR; ./bin/test -slow
     - cd $SRCDIR/experimental; make
     - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup)

experimental/demoprobe/generate.go

@@ -91,12 +91,10 @@ func DemoReport(nodeCount int) report.Report {
 			edgeKeyIngress = report.MakeEdgeID(dstPortID, srcPortID)
 		)
 		r.Endpoint.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{
-			WithConnCountTCP: true,
-			MaxConnCountTCP:  uint(rand.Intn(100) + 10),
+			MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)),
 		}
 		r.Endpoint.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{
-			WithConnCountTCP: true,
-			MaxConnCountTCP:  uint(rand.Intn(100) + 10),
+			MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)),
 		}
 
 		// Address topology
@@ -124,3 +122,5 @@ func DemoReport(nodeCount int) report.Report {
 
 	return r
 }
+
+func newu64(value uint64) *uint64 { return &value }

experimental/genreport/generate.go

@@ -89,12 +89,10 @@ func DemoReport(nodeCount int) report.Report {
 			edgeKeyIngress = report.MakeEdgeID(dstPortID, srcPortID)
 		)
 		r.Endpoint.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{
-			WithConnCountTCP: true,
-			MaxConnCountTCP:  uint(rand.Intn(100) + 10),
+			MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)),
 		}
 		r.Endpoint.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{
-			WithConnCountTCP: true,
-			MaxConnCountTCP:  uint(rand.Intn(100) + 10),
+			MaxConnCountTCP: newu64(uint64(rand.Intn(100) + 10)),
 		}
 
 		// Address topology
@@ -122,3 +120,5 @@ func DemoReport(nodeCount int) report.Report {
 
 	return r
 }
+
+func newu64(value uint64) *uint64 { return &value }

probe/endpoint/reporter.go

@@ -79,15 +79,15 @@ func (r *Reporter) addConnection(rpt *report.Report, c *procspy.Connection) {
 	var (
 		localAddressNodeID  = report.MakeAddressNodeID(r.hostID, c.LocalAddress.String())
 		remoteAddressNodeID = report.MakeAddressNodeID(r.hostID, c.RemoteAddress.String())
-		adjacencyID         = report.MakeAdjacencyID(localAddressNodeID)
+		adjecencyID         = report.MakeAdjacencyID(localAddressNodeID)
 		edgeID              = report.MakeEdgeID(localAddressNodeID, remoteAddressNodeID)
 	)
 
-	rpt.Address.Adjacency[adjacencyID] = rpt.Address.Adjacency[adjacencyID].Add(remoteAddressNodeID)
+	rpt.Address.Adjacency[adjecencyID] = rpt.Address.Adjacency[adjecencyID].Add(remoteAddressNodeID)
 
 	if _, ok := rpt.Address.NodeMetadatas[localAddressNodeID]; !ok {
 		rpt.Address.NodeMetadatas[localAddressNodeID] = report.MakeNodeMetadataWith(map[string]string{
-			"name": r.hostName, // TODO this is ambiguous, be more specific
+			"name": r.hostName,
 			Addr:   c.LocalAddress.String(),
 		})
 	}
@@ -98,11 +98,11 @@ func (r *Reporter) addConnection(rpt *report.Report, c *procspy.Connection) {
 		var (
 			localEndpointNodeID  = report.MakeEndpointNodeID(r.hostID, c.LocalAddress.String(), strconv.Itoa(int(c.LocalPort)))
 			remoteEndpointNodeID = report.MakeEndpointNodeID(r.hostID, c.RemoteAddress.String(), strconv.Itoa(int(c.RemotePort)))
-			adjacencyID          = report.MakeAdjacencyID(localEndpointNodeID)
+			adjecencyID          = report.MakeAdjacencyID(localEndpointNodeID)
 			edgeID               = report.MakeEdgeID(localEndpointNodeID, remoteEndpointNodeID)
 		)
 
-		rpt.Endpoint.Adjacency[adjacencyID] = rpt.Endpoint.Adjacency[adjacencyID].Add(remoteEndpointNodeID)
+		rpt.Endpoint.Adjacency[adjecencyID] = rpt.Endpoint.Adjacency[adjecencyID].Add(remoteEndpointNodeID)
 
 		if _, ok := rpt.Endpoint.NodeMetadatas[localEndpointNodeID]; !ok {
 			// First hit establishes NodeMetadata for scoped local address + port
@@ -119,9 +119,11 @@ func (r *Reporter) addConnection(rpt *report.Report, c *procspy.Connection) {
 	}
 }
 
-func countTCPConnection(m report.EdgeMetadatas, edgeKey string) {
-	edgeMeta := m[edgeKey]
-	edgeMeta.WithConnCountTCP = true
-	edgeMeta.MaxConnCountTCP++
-	m[edgeKey] = edgeMeta
+func countTCPConnection(mds report.EdgeMetadatas, key string) {
+	md := mds[key]
+	if md.MaxConnCountTCP == nil {
+		md.MaxConnCountTCP = new(uint64)
+	}
+	*md.MaxConnCountTCP++
+	mds[key] = md
 }

probe/host/reporter.go

@@ -1,15 +1,14 @@
 package host
 
 import (
-	"net"
 	"runtime"
 	"strings"
 	"time"
 
 	"github.com/weaveworks/scope/report"
 )
 
-// Keys for use in NodeMetadata
+// Keys for use in NodeMetadata.
 const (
 	Timestamp     = "ts"
 	HostName      = "host_name"
@@ -20,30 +19,31 @@ const (
 	Uptime        = "uptime"
 )
 
-// Exposed for testing
+// Exposed for testing.
 const (
 	ProcUptime = "/proc/uptime"
 	ProcLoad   = "/proc/loadavg"
 )
 
-// Exposed for testing
+// Exposed for testing.
 var (
-	InterfaceAddrs = net.InterfaceAddrs
-	Now            = func() string { return time.Now().UTC().Format(time.RFC3339Nano) }
+	Now = func() string { return time.Now().UTC().Format(time.RFC3339Nano) }
 )
 
 // Reporter generates Reports containing the host topology.
 type Reporter struct {
-	hostID   string
-	hostName string
+	hostID    string
+	hostName  string
+	localNets report.Networks
 }
 
 // NewReporter returns a Reporter which produces a report containing host
 // topology for this host.
-func NewReporter(hostID, hostName string) *Reporter {
+func NewReporter(hostID, hostName string, localNets report.Networks) *Reporter {
 	return &Reporter{
-		hostID:   hostID,
-		hostName: hostName,
+		hostID:    hostID,
+		hostName:  hostName,
+		localNets: localNets,
 	}
 }
 
@@ -54,15 +54,8 @@ func (r *Reporter) Report() (report.Report, error) {
 		localCIDRs []string
 	)
 
-	localNets, err := InterfaceAddrs()
-	if err != nil {
-		return rep, err
-	}
-	for _, localNet := range localNets {
-		// Not all networks are IP networks.
-		if ipNet, ok := localNet.(*net.IPNet); ok {
-			localCIDRs = append(localCIDRs, ipNet.String())
-		}
+	for _, localNet := range r.localNets {
+		localCIDRs = append(localCIDRs, localNet.String())
 	}
 
 	uptime, err := GetUptime()

probe/host/reporter_test.go

@@ -12,38 +12,37 @@ import (
 	"github.com/weaveworks/scope/test"
 )
 
-const (
-	release  = "release"
-	version  = "version"
-	network  = "192.168.0.0/16"
-	hostID   = "hostid"
-	now      = "now"
-	hostname = "hostname"
-	load     = "0.59 0.36 0.29"
-	uptime   = "278h55m43s"
-	kernel   = "release version"
-)
-
 func TestReporter(t *testing.T) {
+	var (
+		release     = "release"
+		version     = "version"
+		network     = "192.168.0.0/16"
+		hostID      = "hostid"
+		now         = "now"
+		hostname    = "hostname"
+		load        = "0.59 0.36 0.29"
+		uptime      = "278h55m43s"
+		kernel      = "release version"
+		_, ipnet, _ = net.ParseCIDR(network)
+		localNets   = report.Networks([]*net.IPNet{ipnet})
+	)
+
 	var (
 		oldGetKernelVersion = host.GetKernelVersion
 		oldGetLoad          = host.GetLoad
 		oldGetUptime        = host.GetUptime
-		oldInterfaceAddrs   = host.InterfaceAddrs
 		oldNow              = host.Now
 	)
 	defer func() {
 		host.GetKernelVersion = oldGetKernelVersion
 		host.GetLoad = oldGetLoad
 		host.GetUptime = oldGetUptime
-		host.InterfaceAddrs = oldInterfaceAddrs
 		host.Now = oldNow
 	}()
 	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
 	host.GetLoad = func() string { return load }
 	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
 	host.Now = func() string { return now }
-	host.InterfaceAddrs = func() ([]net.Addr, error) { _, ipnet, _ := net.ParseCIDR(network); return []net.Addr{ipnet}, nil }
 
 	want := report.MakeReport()
 	want.Host.NodeMetadatas[report.MakeHostNodeID(hostID)] = report.MakeNodeMetadataWith(map[string]string{
@@ -55,8 +54,7 @@ func TestReporter(t *testing.T) {
 		host.Uptime:        uptime,
 		host.KernelVersion: kernel,
 	})
-	r := host.NewReporter(hostID, hostname)
-	have, _ := r.Report()
+	have, _ := host.NewReporter(hostID, hostname, localNets).Report()
 	if !reflect.DeepEqual(want, have) {
 		t.Errorf("%s", test.Diff(want, have))
 	}