Concurrent map read/write - intermittent build failure

[peterbroadhurst]

https://github.com/hyperledger/firefly/pull/253/checks?check_run_id=3900059278

time="2021-10-14T21:51:25Z" level=info msg="<-- PUT /api/v1/namespaces/ns1/subscriptions [200] (0.15ms)" httpreq=mYd69rJo
time="2021-10-14T21:51:25Z" level=info msg="api listening on HTTP 127.0.0.1:43683"
time="2021-10-14T21:51:25Z" level=debug msg="CORS origins=[*] methods=[GET POST PUT PATCH DELETE] headers=[*] creds=true maxAge=600"
time="2021-10-14T21:51:25Z" level=info msg="API server context cancelled - shutting down"
time="2021-10-14T21:51:25Z" level=info msg="admin listening on HTTP 127.0.0.1:43931"
time="2021-10-14T21:51:25Z" level=info msg="API server complete"
time="2021-10-14T21:51:25Z" level=debug msg="CORS origins=[*] methods=[GET POST PUT PATCH DELETE] headers=[*] creds=true maxAge=600"
time="2021-10-14T21:51:25Z" level=info msg="API server context cancelled - shutting down"
fatal error: concurrent map read and map write

goroutine 131 [running]:
runtime.throw(0xd09eb9, 0x21)
	/opt/hostedtoolcache/go/1.16.9/x64/src/runtime/panic.go:1117 +0x72 fp=0xc00050dc48 sp=0xc00050dc18 pc=0x4412f2
runtime.mapaccess2_faststr(0xc2c520, 0xc0002a3c80, 0xc00053c108, 0x3, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/runtime/map_faststr.go:116 +0x4a5 fp=0xc00050dcb8 sp=0xc00050dc48 pc=0x41d2e5
github.com/spf13/viper.(*Viper).searchMap(0xc000307b00, 0xc0002a3c80, 0xc0004ce080, 0x2, 0x2, 0x0, 0x0)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:568 +0x5f fp=0xc00050dd00 sp=0xc00050dcb8 pc=0x85be9f
github.com/spf13/viper.(*Viper).find(0xc000307b00, 0xc00053c108, 0x13, 0x1, 0x2, 0x2)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:1218 +0x40e fp=0xc00050de10 sp=0xc00050dd00 pc=0x85dc6e
github.com/spf13/viper.(*Viper).Get(0xc000307b00, 0xcfd565, 0x13, 0x7ee1c5, 0x11e8998)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:798 +0x85 fp=0xc00050dea8 sp=0xc00050de10 pc=0x85cc65
github.com/spf13/viper.(*Viper).GetString(0xc000307b00, 0xcfd565, 0x13, 0xcfd565, 0x13)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:865 +0x3f fp=0xc00050dee8 sp=0xc00050dea8 pc=0x85d47f
github.com/spf13/viper.GetString(...)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:862
github.com/hyperledger/firefly/internal/config.(*configPrefix).GetDuration(0x11e8980, 0xcfd565, 0x13, 0x2c)
	/home/runner/work/firefly/firefly/internal/config/config.go:556 +0x69 fp=0xc00050df20 sp=0xc00050dee8 pc=0x8626c9
github.com/hyperledger/firefly/internal/config.GetDuration(...)
	/home/runner/work/firefly/firefly/internal/config/config.go:553
github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP.func1(0xde8898, 0xc0002e1c40, 0xc00034c700, 0xc000446120)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:175 +0x179 fp=0xc00050dfc0 sp=0xc00050df20 pc=0xac9cd9
runtime.goexit()
	/opt/hostedtoolcache/go/1.16.9/x64/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc00050dfc8 sp=0xc00050dfc0 pc=0x47a441
created by github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:171 +0x98

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000307980, 0xcfa6ea, 0x10, 0xd361d0, 0x49be01)
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1239 +0x2da
testing.runTests.func1(0xc000256300)
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1511 +0x78
testing.tRunner(0xc000256300, 0xc00022fda8)
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1193 +0xef
testing.runTests(0xc00000fe18, 0x11881c0, 0x73, 0x73, 0xc0524779a0b30055, 0x2544d90fe, 0x11e96a0, 0xcf84bf)
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1509 +0x2fe
testing.(*M).Run(0xc0000d9480, 0x0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1417 +0x1eb
main.main()
	_testmain.go:463 +0x1c5

goroutine 130 [select]:
net/http.(*Server).Shutdown(0xc0001f60e0, 0xde8908, 0xc00006c300, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/net/http/server.go:2739 +0x272
github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP.func1(0xde8898, 0xc0002e1c40, 0xc000399730, 0xc000446000)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:177 +0x1dd
created by github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:171 +0x98

goroutine 13 [chan send]:
github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP.func1(0xde8898, 0xc0002b6140, 0xc000134230, 0xc000446000)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:178 +0x228
created by github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:171 +0x98

goroutine 23 [chan send]:
github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP(0xc000127e30, 0xde8898, 0xc000021780)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:198 +0x209
github.com/hyperledger/firefly/internal/apiserver.TestShutdownError.func2(0xc000127e30, 0xde8898, 0xc000021780)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server_test.go:95 +0x3f
created by github.com/hyperledger/firefly/internal/apiserver.TestShutdownError
	/home/runner/work/firefly/firefly/internal/apiserver/http_server_test.go:94 +0x553

goroutine 119 [runnable]:
github.com/spf13/viper.(*Viper).realKey(0xc000307b00, 0xc00034e120, 0x23, 0x23, 0xc00036eab8)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:1357 +0x1e5
github.com/spf13/viper.(*Viper).SetDefault(0xc000307b00, 0xd0c200, 0x23, 0xbf3620, 0xdca880)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:1384 +0x72
github.com/spf13/viper.SetDefault(...)
	/home/runner/go/pkg/mod/github.com/spf13/viper@v1.8.1/viper.go:1380
github.com/hyperledger/firefly/internal/config.Reset()
	/home/runner/work/firefly/firefly/internal/config/config.go:333 +0x11b4
github.com/hyperledger/firefly/internal/apiserver.TestStartAPIFail(0xc000307980)
	/home/runner/work/firefly/firefly/internal/apiserver/server_test.go:79 +0x34
testing.tRunner(0xc000307980, 0xd361d0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1193 +0xef
created by testing.(*T).Run
	/opt/hostedtoolcache/go/1.16.9/x64/src/testing/testing.go:1238 +0x2b3

goroutine 118 [IO wait]:
internal/poll.runtime_pollWait(0x7fcebdf2e730, 0x72, 0x0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0000d9198, 0x72, 0x0, 0x0, 0xcf15e9)
	/opt/hostedtoolcache/go/1.16.9/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.16.9/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0000d9180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc0000d9180, 0x30, 0x30, 0x7fcee8bea108)
	/opt/hostedtoolcache/go/1.16.9/x64/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc000595b00, 0xc0000415e0, 0x417178, 0x30)
	/opt/hostedtoolcache/go/1.16.9/x64/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc000595b00, 0xc9a260, 0xc000316210, 0xc1fe00, 0x116f460)
	/opt/hostedtoolcache/go/1.16.9/x64/src/net/tcpsock.go:261 +0x65
net/http.(*Server).Serve(0xc0001f61c0, 0xde6458, 0xc000595b00, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.9/x64/src/net/http/server.go:2961 +0x285
github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP(0xc00034c700, 0xde8898, 0xc0002e1c40)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:190 +0x293
created by github.com/hyperledger/firefly/internal/apiserver.(*apiServer).Serve
	/home/runner/work/firefly/firefly/internal/apiserver/server.go:99 +0x229

goroutine 117 [runnable]:
github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP(0xc000399730, 0xde8898, 0xc0002e1c40)
	/home/runner/work/firefly/firefly/internal/apiserver/http_server.go:198 +0x209
created by github.com/hyperledger/firefly/internal/apiserver.(*apiServer).Serve
	/home/runner/work/firefly/firefly/internal/apiserver/server.go:91 +0x369
FAIL	github.com/hyperledger/firefly/internal/apiserver	0.608s

[peterbroadhurst]

Also hit it in the e2e:

�[34mfirefly_core_0_1  |�[0m [2021-10-14T22:08:29.423Z]  INFO API server context cancelled - shutting down pid=1
�[34mfirefly_core_0_1  |�[0m fatal error: concurrent map read and map write
�[34mfirefly_core_0_1  |�[0m 
�[34mfirefly_core_0_1  |�[0m goroutine 122 [running]:
�[34mfirefly_core_0_1  |�[0m runtime.throw(0xdd0849, 0x21)
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc000457df0 sp=0xc000457dc0 pc=0x43fa52
�[34mfirefly_core_0_1  |�[0m runtime.mapaccess1_faststr(0xcdd240, 0xc000216870, 0xdc2850, 0x13, 0x13)
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/runtime/map_faststr.go:21 +0x465 fp=0xc000457e60 sp=0xc000457df0 pc=0x41b965
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/config.(*configPrefix).prefixKey(0x132bfe0, 0xdc2850, 0x13, 0x0, 0x0)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/config/config.go:446 +0xe9 fp=0xc000457ee8 sp=0xc000457e60 pc=0x9338a9
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/config.(*configPrefix).GetDuration(0x132bfe0, 0xdc2850, 0x13, 0x2c)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/config/config.go:556 +0x45 fp=0xc000457f20 sp=0xc000457ee8 pc=0x934925
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/config.GetDuration(...)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/config/config.go:553
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP.func1(0xece8d8, 0xc0002b6340, 0xc0000de620, 0xc0002fe660)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/apiserver/http_server.go:175 +0x15f fp=0xc000457fc0 sp=0xc000457f20 pc=0xb4ee5f
�[34mfirefly_core_0_1  |�[0m runtime.goexit()
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc000457fc8 sp=0xc000457fc0 pc=0x479461
�[34mfirefly_core_0_1  |�[0m created by github.com/hyperledger/firefly/internal/apiserver.(*httpServer).serveHTTP
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/apiserver/http_server.go:171 +0x88
�[34mfirefly_core_0_1  |�[0m 
�[34mfirefly_core_0_1  |�[0m goroutine 1 [runnable]:
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/config.(*configPrefix).AddKnownKey(0xc0001c7800, 0xdbd091, 0xd, 0x0, 0x0, 0x0)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/config/config.go:513 +0x110
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/restclient.InitPrefix(0x7ff9670240c0, 0xc0001c7800)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/restclient/config.go:60 +0x143
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/config/wsconfig.InitPrefix(0x7ff9670240c0, 0xc0001c7800)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/config/wsconfig/wsconfig.go:46 +0x39
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/blockchain/ethereum.(*Ethereum).InitPrefix(0x132cec0, 0xed9298, 0xc0001c77c0)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/blockchain/ethereum/config.go:54 +0x9d
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/blockchain/bifactory.InitPrefix(0xed9298, 0xc0001c7e80)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/blockchain/bifactory/factory.go:44 +0xb5
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/orchestrator.NewOrchestrator(0xece980, 0xc0002c82d0)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/orchestrator/orchestrator.go:150 +0x57
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/cmd.getOrchestrator(...)
�[34mfirefly_core_0_1  |�[0m 	/firefly/cmd/firefly.go:81
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/cmd.run(0x0, 0x0)
�[34mfirefly_core_0_1  |�[0m 	/firefly/cmd/firefly.go:116 +0x6fa
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/cmd.glob..func1(0x12c7e20, 0x135c6b8, 0x0, 0x0, 0x0, 0x0)
�[34mfirefly_core_0_1  |�[0m 	/firefly/cmd/firefly.go:48 +0x25
�[34mfirefly_core_0_1  |�[0m github.com/spf13/cobra.(*Command).execute(0x12c7e20, 0xc000020220, 0x0, 0x0, 0x12c7e20, 0xc000020220)
�[34mfirefly_core_0_1  |�[0m 	/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:852 +0x472
�[34mfirefly_core_0_1  |�[0m github.com/spf13/cobra.(*Command).ExecuteC(0x12c7e20, 0x44a0dc, 0xc000000180, 0x300000002)
�[34mfirefly_core_0_1  |�[0m 	/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960 +0x375
�[34mfirefly_core_0_1  |�[0m github.com/spf13/cobra.(*Command).Execute(...)
�[34mfirefly_core_0_1  |�[0m 	/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/cmd.Execute(0x4746c5, 0xc000000180)
�[34mfirefly_core_0_1  |�[0m 	/firefly/cmd/firefly.go:87 +0x35
�[34mfirefly_core_0_1  |�[0m main.main()
�[34mfirefly_core_0_1  |�[0m 	/firefly/main.go:27 +0x26
�[34mfirefly_core_0_1  |�[0m 
�[34mfirefly_core_0_1  |�[0m goroutine 18 [syscall]:
�[34mfirefly_core_0_1  |�[0m os/signal.signal_recv(0x0)
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/runtime/sigqueue.go:168 +0xa5
�[34mfirefly_core_0_1  |�[0m os/signal.loop()
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
�[34mfirefly_core_0_1  |�[0m created by os/signal.Notify.func1.1
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/os/signal/signal.go:151 +0x45
�[34mfirefly_core_0_1  |�[0m 
�[34mfirefly_core_0_1  |�[0m goroutine 7 [select]:
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/apiserver.(*apiServer).waitForServerStop(0xc0002b9da0, 0xc00033c840, 0xc00033c8a0, 0xece8d8, 0xc0002b6340)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/apiserver/server.go:106 +0xa5
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/internal/apiserver.(*apiServer).Serve(0xc0002b9da0, 0xece8d8, 0xc0002b6340, 0xedb788, 0xc000260600, 0x0, 0x1)
�[34mfirefly_core_0_1  |�[0m 	/firefly/internal/apiserver/server.go:102 +0xe5
�[34mfirefly_core_0_1  |�[0m github.com/hyperledger/firefly/cmd.startFirefly(0xece8d8, 0xc0002b6340, 0xc00020bb40, 0xedb788, 0xc000260600, 0xebe800, 0xc0002b9da0, 0xc000071b00)
�[34mfirefly_core_0_1  |�[0m 	/firefly/cmd/firefly.go:163 +0x134
�[34mfirefly_core_0_1  |�[0m created by github.com/hyperledger/firefly/cmd.run
�[34mfirefly_core_0_1  |�[0m 	/firefly/cmd/firefly.go:118 +0x475
�[34mfirefly_core_0_1  |�[0m 
�[34mfirefly_core_0_1  |�[0m goroutine 8 [IO wait]:
�[34mfirefly_core_0_1  |�[0m internal/poll.runtime_pollWait(0x7ff967023f78, 0x72, 0x0)
�[34mfirefly_core_0_1  |�[0m 	/usr/local/go/src/runtime/netpoll.go:222 +0x55
�[35methconnect_1_1    |�[0m Copyright (C) 2018,2021 Kaleido
�[35methconnect_1_1    |�[0m Licensed under the Apache License, Version 2.0
�[35methconnect_1_1    |�[0m Version:  (Build Date: 2021-10-13T12:03:18Z)
�[35methconnect_1_1    |�[0m 
�[35methconnect_1_1    |�[0m [2021-10-14T22:07:53.750Z] DEBUG Log level set to 3