Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fatal error: concurrent map iteration and map write #15705

Closed
prezha opened this issue Jan 25, 2023 · 0 comments · Fixed by #15706
Closed

fatal error: concurrent map iteration and map write #15705

prezha opened this issue Jan 25, 2023 · 0 comments · Fixed by #15706
Labels
kind/bug Categorizes issue or PR as related to a bug. priority/important-soon Must be staffed and worked on either currently, or very soon, ideally in time for the next release.

Comments

@prezha
Copy link
Contributor

prezha commented Jan 25, 2023

What Happened?

as seen eg, in https://storage.googleapis.com/minikube-builds/logs/15695/27561/Docker_Linux_containerd.html#fail_TestMultiNode%2fserial%2fStartAfterStop, we can hit a race condition over addons map concurrent reading & writing, with any combination of os/driver/container runtime:

	I0124 02:43:04.247029  114443 addons.go:486] enableAddons start: toEnable=map[default-storageclass:true storage-provisioner:true], additional=[]
	fatal error: concurrent map iteration and map write
	
	goroutine 1 [running]:
	reflect.mapiternext(0x4ab46f?)
		/usr/local/go/src/runtime/map.go:1380 +0x19
	reflect.(*MapIter).Next(0xc000d99ed8?)
		/usr/local/go/src/reflect/value.go:1846 +0x7e
	internal/fmtsort.Sort({0x200ede0?, 0xc000054eb0?, 0xc000c480a0?})
		/usr/local/go/src/internal/fmtsort/sort.go:62 +0x1f3
	fmt.(*pp).printValue(0xc0004625b0, {0x200ede0?, 0xc000054eb0?, 0x0?}, 0x76, 0x2)
		/usr/local/go/src/fmt/print.go:800 +0xa0b
	fmt.(*pp).printValue(0xc0004625b0, {0x235c740?, 0xc000054b00?, 0xc000d9a428?}, 0x76, 0x1)
		/usr/local/go/src/fmt/print.go:837 +0x12b4
	fmt.(*pp).printValue(0xc0004625b0, {0x1eb73e0?, 0xc000054b00?, 0x3?}, 0x76, 0x0)
		/usr/local/go/src/fmt/print.go:907 +0xb85
	fmt.(*pp).printArg(0xc0004625b0, {0x1eb73e0?, 0xc000054b00}, 0x76)
		/usr/local/go/src/fmt/print.go:743 +0x756
	fmt.(*pp).doPrintf(0xc0004625b0, {0x2381664, 0x10}, {0xc0003f6c90?, 0x1, 0x1})
		/usr/local/go/src/fmt/print.go:1057 +0x288
	fmt.Fprintf({0x29f94a0, 0xc0004b8000}, {0x2381664, 0x10}, {0xc0003f6c90, 0x1, 0x1})
		/usr/local/go/src/fmt/print.go:204 +0x75
	k8s.io/klog/v2.(*loggingT).printfDepth(0x10?, 0x0?, 0x0, {0x0, 0x0}, 0x83?, {0x2381664, 0x10}, {0xc0003f6c90, 0x1, ...})
		/go/pkg/mod/k8s.io/klog/v2@v2.90.0/klog.go:727 +0x165
	k8s.io/klog/v2.(*loggingT).printf(...)
		/go/pkg/mod/k8s.io/klog/v2@v2.90.0/klog.go:713
	k8s.io/klog/v2.Infof(...)
		/go/pkg/mod/k8s.io/klog/v2@v2.90.0/klog.go:1470
	k8s.io/minikube/pkg/minikube/node.joinCluster({{0x2a1a360, 0xc000c4cd40}, 0x1, 0x0, {0x2a1e9a0, 0xc000a0d4c0}, 0xc000a53920, 0xc000054b00, 0xc000dbb950, 0xc00077dd40}, ...)
		/app/pkg/minikube/node/start.go:288 +0x111
	k8s.io/minikube/pkg/minikube/node.Start({{0x2a1a360, 0xc000c4cd40}, 0x1, 0x0, {0x2a1e9a0, 0xc000a0d4c0}, 0xc000a53920, 0xc000054b00, 0xc000dbb950, 0xc00077dd40}, ...)
		/app/pkg/minikube/node/start.go:200 +0xba5
	k8s.io/minikube/cmd/minikube/cmd.glob..func31(0x3b22d00?, {0xc000e01180, 0x1, 0x4?})
		/app/cmd/minikube/cmd/node_start.go:74 +0x4f5
	github.com/spf13/cobra.(*Command).execute(0x3b22d00, {0xc000e01140, 0x4, 0x4})
		/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:920 +0x847
	github.com/spf13/cobra.(*Command).ExecuteC(0x3b20a80)
		/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044 +0x3bd
	github.com/spf13/cobra.(*Command).Execute(...)
		/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968
	k8s.io/minikube/cmd/minikube/cmd.Execute()
		/app/cmd/minikube/cmd/root.go:173 +0xa06
	main.main()
		/app/cmd/minikube/main.go:95 +0x25d
	
	goroutine 8 [select]:
	go.opencensus.io/stats/view.(*worker).start(0xc00050c100)
		/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0xad
	created by go.opencensus.io/stats/view.init.0
		/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x8d
	
	goroutine 291 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleGlobalRequests(0x0?, 0x0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:142 +0x57
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:59 +0xe5
	
	goroutine 143 [select]:
	golang.org/x/crypto/ssh.(*handshakeTransport).kexLoop(0xc000eb0480)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:270 +0x4fa
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:137 +0x236
	
	goroutine 290 [chan receive]:
	golang.org/x/crypto/ssh.(*handshakeTransport).readPacket(0xc000eb0180)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:189 +0x39
	golang.org/x/crypto/ssh.(*mux).onePacket(0xc0001ad260)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:215 +0x2d
	golang.org/x/crypto/ssh.(*mux).loop(0xc0001ad260)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:190 +0x28
	created by golang.org/x/crypto/ssh.newMux
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:128 +0x190
	
	goroutine 252 [IO wait]:
	internal/poll.runtime_pollWait(0x7fcee07f2aa0, 0x72)
		/usr/local/go/src/runtime/netpoll.go:305 +0x89
	internal/poll.(*pollDesc).wait(0xc00014ee80?, 0xc00066c000?, 0x0)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
	internal/poll.(*pollDesc).waitRead(...)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
	internal/poll.(*FD).Read(0xc00014ee80, {0xc00066c000, 0x1000, 0x1000})
		/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
	net.(*netFD).Read(0xc00014ee80, {0xc00066c000?, 0x10?, 0x12c0?})
		/usr/local/go/src/net/fd_posix.go:55 +0x29
	net.(*conn).Read(0xc000772030, {0xc00066c000?, 0x10?, 0x12c0?})
		/usr/local/go/src/net/net.go:183 +0x45
	bufio.(*Reader).Read(0xc0010ea180, {0xc000a103d0, 0x4, 0xc0000c9900?})
		/usr/local/go/src/bufio/bufio.go:237 +0x1bb
	io.ReadAtLeast({0x29f2640, 0xc0010ea180}, {0xc000a103d0, 0x4, 0x4}, 0x4)
		/usr/local/go/src/io/io.go:332 +0x9a
	io.ReadFull(...)
		/usr/local/go/src/io/io.go:351
	golang.org/x/crypto/ssh.(*gcmCipher).readCipherPacket(0xc000a103c0, 0x0?, {0x29f2640, 0xc0010ea180})
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/cipher.go:372 +0x65
	golang.org/x/crypto/ssh.(*connectionState).readPacket(0xc000614120, 0x3?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:130 +0x42
	golang.org/x/crypto/ssh.(*transport).readPacket(0xc000614120)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:114 +0x2c
	golang.org/x/crypto/ssh.(*handshakeTransport).readOnePacket(0xc000eb0180, 0x0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:377 +0x3d
	golang.org/x/crypto/ssh.(*handshakeTransport).readLoop(0xc000eb0180)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:199 +0x45
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:136 +0x1f6
	
	goroutine 255 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleGlobalRequests(0xa?, 0x10?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:142 +0x57
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:59 +0xe5
	
	goroutine 293 [sync.Cond.Wait]:
	sync.runtime_notifyListWait(0xc00006fad0, 0x0)
		/usr/local/go/src/runtime/sema.go:517 +0x14c
	sync.(*Cond).Wait(0x0?)
		/usr/local/go/src/sync/cond.go:70 +0x8c
	golang.org/x/crypto/ssh.(*mux).Wait(0xc0001ad260)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:110 +0x99
	golang.org/x/crypto/ssh.NewClient.func1()
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:62 +0x2a
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:61 +0x176
	
	goroutine 292 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleChannelOpens(0xc0000c44b0, 0x0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:151 +0x4d
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:60 +0x139
	
	goroutine 153 [select]:
	k8s.io/klog/v2.(*flushDaemon).run.func1()
		/go/pkg/mod/k8s.io/klog/v2@v2.90.0/klog.go:1130 +0x11e
	created by k8s.io/klog/v2.(*flushDaemon).run
		/go/pkg/mod/k8s.io/klog/v2@v2.90.0/klog.go:1126 +0x17b
	
	goroutine 247 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleGlobalRequests(0x5cb8aa?, 0xc000574090?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:142 +0x57
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:59 +0xe5
	
	goroutine 142 [IO wait]:
	internal/poll.runtime_pollWait(0x7fcee07f3040, 0x72)
		/usr/local/go/src/runtime/netpoll.go:305 +0x89
	internal/poll.(*pollDesc).wait(0xc000cf2780?, 0xc000efa000?, 0x0)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
	internal/poll.(*pollDesc).waitRead(...)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
	internal/poll.(*FD).Read(0xc000cf2780, {0xc000efa000, 0x1000, 0x1000})
		/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
	net.(*netFD).Read(0xc000cf2780, {0xc000efa000?, 0x10?, 0x270?})
		/usr/local/go/src/net/fd_posix.go:55 +0x29
	net.(*conn).Read(0xc00072a598, {0xc000efa000?, 0x10?, 0x270?})
		/usr/local/go/src/net/net.go:183 +0x45
	bufio.(*Reader).Read(0xc0009e9bc0, {0xc00045b280, 0x4, 0xc0003c6780?})
		/usr/local/go/src/bufio/bufio.go:237 +0x1bb
	io.ReadAtLeast({0x29f2640, 0xc0009e9bc0}, {0xc00045b280, 0x4, 0x4}, 0x4)
		/usr/local/go/src/io/io.go:332 +0x9a
	io.ReadFull(...)
		/usr/local/go/src/io/io.go:351
	golang.org/x/crypto/ssh.(*gcmCipher).readCipherPacket(0xc00045b270, 0x0?, {0x29f2640, 0xc0009e9bc0})
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/cipher.go:372 +0x65
	golang.org/x/crypto/ssh.(*connectionState).readPacket(0xc0003bb560, 0x3?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:130 +0x42
	golang.org/x/crypto/ssh.(*transport).readPacket(0xc0003bb560)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:114 +0x2c
	golang.org/x/crypto/ssh.(*handshakeTransport).readOnePacket(0xc000eb0480, 0x0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:377 +0x3d
	golang.org/x/crypto/ssh.(*handshakeTransport).readLoop(0xc000eb0480)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:199 +0x45
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:136 +0x1f6
	
	goroutine 210 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleGlobalRequests(0xc0000c4e80?, 0xc0000c4e60?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:142 +0x57
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:59 +0xe5
	
	goroutine 248 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleChannelOpens(0xc00045b4a0, 0x0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:151 +0x4d
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:60 +0x139
	
	goroutine 249 [sync.Cond.Wait]:
	sync.runtime_notifyListWait(0xc000d49790, 0x0)
		/usr/local/go/src/runtime/sema.go:517 +0x14c
	sync.(*Cond).Wait(0xc000064f00?)
		/usr/local/go/src/sync/cond.go:70 +0x8c
	golang.org/x/crypto/ssh.(*mux).Wait(0xc0004b8150)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:110 +0x99
	golang.org/x/crypto/ssh.NewClient.func1()
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:62 +0x2a
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:61 +0x176
	
	goroutine 257 [sync.Cond.Wait]:
	sync.runtime_notifyListWait(0xc00006f810, 0x0)
		/usr/local/go/src/runtime/sema.go:517 +0x14c
	sync.(*Cond).Wait(0x0?)
		/usr/local/go/src/sync/cond.go:70 +0x8c
	golang.org/x/crypto/ssh.(*mux).Wait(0xc0001ad180)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:110 +0x99
	golang.org/x/crypto/ssh.NewClient.func1()
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:62 +0x2a
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:61 +0x176
	
	goroutine 193 [chan receive]:
	golang.org/x/crypto/ssh.(*handshakeTransport).readPacket(0xc000eb0480)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:189 +0x39
	golang.org/x/crypto/ssh.(*mux).onePacket(0xc000482930)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:215 +0x2d
	golang.org/x/crypto/ssh.(*mux).loop(0xc000482930)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:190 +0x28
	created by golang.org/x/crypto/ssh.newMux
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:128 +0x190
	
	goroutine 211 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleChannelOpens(0xc000742870, 0xc0001bb8f0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:151 +0x4d
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:60 +0x139
	
	goroutine 245 [select]:
	golang.org/x/crypto/ssh.(*handshakeTransport).kexLoop(0xc000eb0780)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:270 +0x4fa
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:137 +0x236
	
	goroutine 246 [chan receive]:
	golang.org/x/crypto/ssh.(*handshakeTransport).readPacket(0xc000eb0780)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:189 +0x39
	golang.org/x/crypto/ssh.(*mux).onePacket(0xc0004b8150)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:215 +0x2d
	golang.org/x/crypto/ssh.(*mux).loop(0xc0004b8150)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:190 +0x28
	created by golang.org/x/crypto/ssh.newMux
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:128 +0x190
	
	goroutine 212 [sync.Cond.Wait]:
	sync.runtime_notifyListWait(0xc000e01b90, 0x0)
		/usr/local/go/src/runtime/sema.go:517 +0x14c
	sync.(*Cond).Wait(0x0?)
		/usr/local/go/src/sync/cond.go:70 +0x8c
	golang.org/x/crypto/ssh.(*mux).Wait(0xc000482930)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:110 +0x99
	golang.org/x/crypto/ssh.NewClient.func1()
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:62 +0x2a
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:61 +0x176
	
	goroutine 218 [IO wait]:
	internal/poll.runtime_pollWait(0x7fcee07f3310, 0x72)
		/usr/local/go/src/runtime/netpoll.go:305 +0x89
	internal/poll.(*pollDesc).wait(0xc000727c00?, 0xc000fa6000?, 0x0)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
	internal/poll.(*pollDesc).waitRead(...)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
	internal/poll.(*FD).Read(0xc000727c00, {0xc000fa6000, 0x1000, 0x1000})
		/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
	net.(*netFD).Read(0xc000727c00, {0xc000fa6000?, 0x10?, 0x270?})
		/usr/local/go/src/net/fd_posix.go:55 +0x29
	net.(*conn).Read(0xc0005fade0, {0xc000fa6000?, 0x10?, 0x270?})
		/usr/local/go/src/net/net.go:183 +0x45
	bufio.(*Reader).Read(0xc000f9c300, {0xc0007429c0, 0x4, 0xc0003c6c80?})
		/usr/local/go/src/bufio/bufio.go:237 +0x1bb
	io.ReadAtLeast({0x29f2640, 0xc000f9c300}, {0xc0007429c0, 0x4, 0x4}, 0x4)
		/usr/local/go/src/io/io.go:332 +0x9a
	io.ReadFull(...)
		/usr/local/go/src/io/io.go:351
	golang.org/x/crypto/ssh.(*gcmCipher).readCipherPacket(0xc0007429b0, 0x0?, {0x29f2640, 0xc000f9c300})
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/cipher.go:372 +0x65
	golang.org/x/crypto/ssh.(*connectionState).readPacket(0xc0009c0b40, 0x3?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:130 +0x42
	golang.org/x/crypto/ssh.(*transport).readPacket(0xc0009c0b40)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:114 +0x2c
	golang.org/x/crypto/ssh.(*handshakeTransport).readOnePacket(0xc0001c9980, 0x0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:377 +0x3d
	golang.org/x/crypto/ssh.(*handshakeTransport).readLoop(0xc0001c9980)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:199 +0x45
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:136 +0x1f6
	
	goroutine 238 [select]:
	golang.org/x/crypto/ssh.(*handshakeTransport).kexLoop(0xc0001c9380)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:270 +0x4fa
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:137 +0x236
	
	goroutine 219 [select]:
	golang.org/x/crypto/ssh.(*handshakeTransport).kexLoop(0xc0001c9980)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:270 +0x4fa
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:137 +0x236
	
	goroutine 220 [chan receive]:
	golang.org/x/crypto/ssh.(*handshakeTransport).readPacket(0xc0001c9980)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:189 +0x39
	golang.org/x/crypto/ssh.(*mux).onePacket(0xc000482af0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:215 +0x2d
	golang.org/x/crypto/ssh.(*mux).loop(0xc000482af0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:190 +0x28
	created by golang.org/x/crypto/ssh.newMux
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:128 +0x190
	
	goroutine 221 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleGlobalRequests(0x0?, 0x0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:142 +0x57
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:59 +0xe5
	
	goroutine 222 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleChannelOpens(0xc000742aa0, 0x0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:151 +0x4d
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:60 +0x139
	
	goroutine 223 [sync.Cond.Wait]:
	sync.runtime_notifyListWait(0xc00019e010, 0x0)
		/usr/local/go/src/runtime/sema.go:517 +0x14c
	sync.(*Cond).Wait(0x0?)
		/usr/local/go/src/sync/cond.go:70 +0x8c
	golang.org/x/crypto/ssh.(*mux).Wait(0xc000482af0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:110 +0x99
	golang.org/x/crypto/ssh.NewClient.func1()
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:62 +0x2a
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:61 +0x176
	
	goroutine 244 [IO wait]:
	internal/poll.runtime_pollWait(0x7fcee07f3220, 0x72)
		/usr/local/go/src/runtime/netpoll.go:305 +0x89
	internal/poll.(*pollDesc).wait(0xc000cf2c80?, 0xc000eff000?, 0x0)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
	internal/poll.(*pollDesc).waitRead(...)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
	internal/poll.(*FD).Read(0xc000cf2c80, {0xc000eff000, 0x1000, 0x1000})
		/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
	net.(*netFD).Read(0xc000cf2c80, {0xc000eff000?, 0x10?, 0x270?})
		/usr/local/go/src/net/fd_posix.go:55 +0x29
	net.(*conn).Read(0xc00072a5c0, {0xc000eff000?, 0x10?, 0x270?})
		/usr/local/go/src/net/net.go:183 +0x45
	bufio.(*Reader).Read(0xc001110240, {0xc00045b3c0, 0x4, 0xc001156500?})
		/usr/local/go/src/bufio/bufio.go:237 +0x1bb
	io.ReadAtLeast({0x29f2640, 0xc001110240}, {0xc00045b3c0, 0x4, 0x4}, 0x4)
		/usr/local/go/src/io/io.go:332 +0x9a
	io.ReadFull(...)
		/usr/local/go/src/io/io.go:351
	golang.org/x/crypto/ssh.(*gcmCipher).readCipherPacket(0xc00045b3b0, 0x0?, {0x29f2640, 0xc001110240})
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/cipher.go:372 +0x65
	golang.org/x/crypto/ssh.(*connectionState).readPacket(0xc0003bb8c0, 0x3?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:130 +0x42
	golang.org/x/crypto/ssh.(*transport).readPacket(0xc0003bb8c0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:114 +0x2c
	golang.org/x/crypto/ssh.(*handshakeTransport).readOnePacket(0xc000eb0780, 0x0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:377 +0x3d
	golang.org/x/crypto/ssh.(*handshakeTransport).readLoop(0xc000eb0780)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:199 +0x45
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:136 +0x1f6
	
	goroutine 256 [chan receive]:
	golang.org/x/crypto/ssh.(*Client).handleChannelOpens(0xc0000c4410, 0x0?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:151 +0x4d
	created by golang.org/x/crypto/ssh.NewClient
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/client.go:60 +0x139
	
	goroutine 254 [chan receive]:
	golang.org/x/crypto/ssh.(*handshakeTransport).readPacket(0xc0001c9380)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:189 +0x39
	golang.org/x/crypto/ssh.(*mux).onePacket(0xc0001ad180)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:215 +0x2d
	golang.org/x/crypto/ssh.(*mux).loop(0xc0001ad180)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:190 +0x28
	created by golang.org/x/crypto/ssh.newMux
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/mux.go:128 +0x190
	
	goroutine 253 [select]:
	golang.org/x/crypto/ssh.(*handshakeTransport).kexLoop(0xc000eb0180)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:270 +0x4fa
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:137 +0x236
	
	goroutine 237 [IO wait]:
	internal/poll.runtime_pollWait(0x7fcee07f2d70, 0x72)
		/usr/local/go/src/runtime/netpoll.go:305 +0x89
	internal/poll.(*pollDesc).wait(0xc000760e80?, 0xc0005c5000?, 0x0)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
	internal/poll.(*pollDesc).waitRead(...)
		/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
	internal/poll.(*FD).Read(0xc000760e80, {0xc0005c5000, 0x1000, 0x1000})
		/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
	net.(*netFD).Read(0xc000760e80, {0xc0005c5000?, 0x10?, 0x270?})
		/usr/local/go/src/net/fd_posix.go:55 +0x29
	net.(*conn).Read(0xc00065a1a8, {0xc0005c5000?, 0x10?, 0x270?})
		/usr/local/go/src/net/net.go:183 +0x45
	bufio.(*Reader).Read(0xc001110420, {0xc000a10290, 0x4, 0xc000579180?})
		/usr/local/go/src/bufio/bufio.go:237 +0x1bb
	io.ReadAtLeast({0x29f2640, 0xc001110420}, {0xc000a10290, 0x4, 0x4}, 0x4)
		/usr/local/go/src/io/io.go:332 +0x9a
	io.ReadFull(...)
		/usr/local/go/src/io/io.go:351
	golang.org/x/crypto/ssh.(*gcmCipher).readCipherPacket(0xc000a10280, 0x0?, {0x29f2640, 0xc001110420})
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/cipher.go:372 +0x65
	golang.org/x/crypto/ssh.(*connectionState).readPacket(0xc0009706c0, 0x3?)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:130 +0x42
	golang.org/x/crypto/ssh.(*transport).readPacket(0xc0009706c0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/transport.go:114 +0x2c
	golang.org/x/crypto/ssh.(*handshakeTransport).readOnePacket(0xc0001c9380, 0x0)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:377 +0x3d
	golang.org/x/crypto/ssh.(*handshakeTransport).readLoop(0xc0001c9380)
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:199 +0x45
	created by golang.org/x/crypto/ssh.newClientTransport
		/go/pkg/mod/golang.org/x/crypto@v0.5.0/ssh/handshake.go:136 +0x1f6
	
	goroutine 560 [runnable]:
	k8s.io/minikube/pkg/addons.Start(0xc0010b6a98?, 0xc000054b00, 0xc00077dd40, {0x3b83668?, 0x0, 0x0})
		/app/pkg/addons/addons.go:519 +0x3bb
	created by k8s.io/minikube/pkg/minikube/node.Start
		/app/pkg/minikube/node/start.go:179 +0xa2a

Attach the log file

Docker_Linux_containerd.log

Operating System

None

Driver

None
@prezha prezha mentioned this issue Jan 25, 2023
Merged
@medyagh medyagh added kind/bug Categorizes issue or PR as related to a bug. priority/important-soon Must be staffed and worked on either currently, or very soon, ideally in time for the next release. labels Jan 25, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
kind/bug Categorizes issue or PR as related to a bug. priority/important-soon Must be staffed and worked on either currently, or very soon, ideally in time for the next release.
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

fix addons map race condition prezha/minikube
2 participants
@medyagh @prezha