Skip to content

Commit

Permalink
agent: Fix data race when accessing d.monitorAgent
Browse files Browse the repository at this point in the history 
Fixes:
```
2020-04-15T08:20:51.171924491Z WARNING: DATA RACE
2020-04-15T08:20:51.171929471Z Write at 0x00c0002d89c0 by main goroutine:
2020-04-15T08:20:51.171932068Z   github.com/cilium/cilium/daemon/cmd.NewDaemon()
2020-04-15T08:20:51.171934565Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon.go:521 +0x3811
2020-04-15T08:20:51.171937059Z   github.com/cilium/cilium/daemon/cmd.(*Daemon).init()
2020-04-15T08:20:51.171939762Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon.go:210 +0x4a4
2020-04-15T08:20:51.171942223Z   github.com/cilium/cilium/daemon/cmd.NewDaemon()
2020-04-15T08:20:51.171944604Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon.go:508 +0x2fa5
2020-04-15T08:20:51.171947111Z   github.com/cilium/cilium/daemon/cmd.NewDaemon()
2020-04-15T08:20:51.171949522Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon.go:411 +0x463e
2020-04-15T08:20:51.171951975Z   github.com/cilium/cilium/daemon/cmd.runDaemon()
2020-04-15T08:20:51.171954378Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon_main.go:1194 +0x345
2020-04-15T08:20:51.171956805Z   github.com/cilium/cilium/daemon/cmd.glob..func1()
2020-04-15T08:20:51.171959248Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon_main.go:107 +0x108
2020-04-15T08:20:51.171961654Z   github.com/cilium/cilium/daemon/cmd.initEnv()
2020-04-15T08:20:51.171964065Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon_main.go:1073 +0x2c56
2020-04-15T08:20:51.171966471Z   github.com/cilium/cilium/daemon/cmd.glob..func1()
2020-04-15T08:20:51.171968921Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon_main.go:105 +0xee
2020-04-15T08:20:51.171971339Z   github.com/spf13/cobra.(*Command).execute()
2020-04-15T08:20:51.171973711Z       /go/src/github.com/cilium/cilium/vendor/github.com/spf13/cobra/command.go:830 +0x8e0
2020-04-15T08:20:51.171983921Z   github.com/spf13/cobra.(*Command).ExecuteC()
2020-04-15T08:20:51.17198653Z       /go/src/github.com/cilium/cilium/vendor/github.com/spf13/cobra/command.go:914 +0x41a
2020-04-15T08:20:51.171989059Z   github.com/spf13/cobra.(*Command).Execute()
2020-04-15T08:20:51.171991461Z       /go/src/github.com/cilium/cilium/vendor/github.com/spf13/cobra/command.go:864 +0x251
2020-04-15T08:20:51.171993892Z   github.com/cilium/cilium/daemon/cmd.Execute()
2020-04-15T08:20:51.171996301Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon_main.go:138 +0x232
2020-04-15T08:20:51.171998765Z   main.main()
2020-04-15T08:20:51.172001107Z       /go/src/github.com/cilium/cilium/daemon/main.go:22 +0x2f
2020-04-15T08:20:51.172003564Z
2020-04-15T08:20:51.172005844Z Previous read at 0x00c0002d89c0 by goroutine 82:
2020-04-15T08:20:51.172008245Z   github.com/cilium/cilium/daemon/cmd.(*Daemon).SendNotification()
2020-04-15T08:20:51.172010648Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon.go:633 +0xc0
2020-04-15T08:20:51.172013085Z   github.com/cilium/cilium/pkg/datapath/ipcache.(*BPFListener).notifyMonitor()
2020-04-15T08:20:51.172015524Z       /go/src/github.com/cilium/cilium/pkg/datapath/ipcache/listener.go:113 +0x288
2020-04-15T08:20:51.172019826Z   github.com/cilium/cilium/pkg/datapath/ipcache.(*BPFListener).OnIPIdentityCacheChange()
2020-04-15T08:20:51.172022388Z       /go/src/github.com/cilium/cilium/pkg/datapath/ipcache/listener.go:142 +0x21e
2020-04-15T08:20:51.172024963Z   github.com/cilium/cilium/pkg/ipcache.(*IPCache).Upsert()
2020-04-15T08:20:51.172027361Z       /go/src/github.com/cilium/cilium/pkg/ipcache/ipcache.go:278 +0xaa0
2020-04-15T08:20:51.172029767Z   github.com/cilium/cilium/daemon/cmd.(*Daemon).syncEndpointsAndHostIPs()
2020-04-15T08:20:51.17203218Z       /go/src/github.com/cilium/cilium/daemon/cmd/datapath.go:275 +0xcd0
2020-04-15T08:20:51.172034647Z   github.com/cilium/cilium/daemon/cmd.(*Daemon).init.func2()
2020-04-15T08:20:51.17203703Z       /go/src/github.com/cilium/cilium/daemon/cmd/daemon.go:224 +0x41
2020-04-15T08:20:51.172039455Z   github.com/cilium/cilium/pkg/controller.(*Controller).runController()
2020-04-15T08:20:51.172041885Z       /go/src/github.com/cilium/cilium/pkg/controller/controller.go:205 +0xc71
```

Fixes: #10987

Signed-off-by: Thomas Graf <thomas@cilium.io>
  • Loading branch information
@tgraf @borkmann
tgraf authored and borkmann committed Jun 2, 2020
1 parent 9f2a0ad commit 870894a
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 16 deletions.
33 changes: 17 additions & 16 deletions daemon/cmd/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,22 +212,6 @@ func (d *Daemon) init() error {
if err := d.Datapath().Loader().Reinitialize(d.ctx, d, d.mtuConfig.GetDeviceMTU(), d.Datapath(), d.l7Proxy, d.ipam); err != nil {
return err
}

if err := d.syncEndpointsAndHostIPs(); err != nil {
return err
}

// Start the controller for periodic sync. The purpose of the
// controller is to ensure that endpoints and host IPs entries are
// reinserted to the bpf maps if they are ever removed from them.
controller.NewManager().UpdateController("sync-endpoints-and-host-ips",
controller.ControllerParams{
DoFunc: func(ctx context.Context) error {
return d.syncEndpointsAndHostIPs()
},
RunInterval: time.Minute,
Context: d.ctx,
})
}

return nil
Expand Down Expand Up @@ -524,6 +508,23 @@ func NewDaemon(ctx context.Context, dp datapath.Datapath) (*Daemon, *endpointRes
}
d.monitorAgent = monitorAgent
}

if err := d.syncEndpointsAndHostIPs(); err != nil {
return nil, nil, err
}

// Start the controller for periodic sync. The purpose of the
// controller is to ensure that endpoints and host IPs entries are
// reinserted to the bpf maps if they are ever removed from them.
controller.NewManager().UpdateController("sync-endpoints-and-host-ips",
controller.ControllerParams{
DoFunc: func(ctx context.Context) error {
return d.syncEndpointsAndHostIPs()
},
RunInterval: time.Minute,
Context: d.ctx,
})

if err := loader.RestoreTemplates(option.Config.StateDir); err != nil {
log.WithError(err).Error("Unable to restore previous BPF templates")
}
Expand Down
4 changes: 4 additions & 0 deletions daemon/cmd/datapath.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ func endParallelMapMode() {
// ipcache, if needed, and also notifies the daemon and network policy
// hosts cache if changes were made.
func (d *Daemon) syncEndpointsAndHostIPs() error {
if option.Config.DryMode {
return nil
}

specialIdentities := []identity.IPIdentityPair{}

if option.Config.EnableIPv4 {
Expand Down

0 comments on commit 870894a

Please sign in to comment.