HTTP pod auto restarting a lot

[nicolashmln]

When we look at the resources (CPU and RAM) everything is fine, the graphs show an almost straight line, but the pod can restart 40 times a day. Here the logs which can be related:

fatal error: concurrent map writes
 
goroutine 72941 [running]:
runtime.throw(0xabf0a8, 0x15)
/usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc000c3b990 sp=0xc000c3b960 pc=0x42f312
runtime.mapassign_faststr(0x9fa100, 0xc000263650, 0xc001c32d00, 0x14, 0xc05937440e1a6e6c)
/usr/local/go/src/runtime/map_faststr.go:291 +0x3fe fp=0xc000c3b9f8 sp=0xc000c3b990 pc=0x41495e
openreplay/backend/pkg/db/cache.(*PGCache).GetProjectByKey(0xc000263680, 0xc001c32d00, 0x14, 0x0, 0x0, 0xa4f000)
/root/pkg/db/cache/project.go:17 +0x176 fp=0xc000c3ba68 sp=0xc000c3b9f8 pc=0x957776
main.startSessionHandlerWeb(0xcbe7a0, 0xc000272620, 0xc001b4a900)
/root/services/http/handlers.go:59 +0x1cc fp=0xc000c3bb80 sp=0xc000c3ba68 pc=0x98bb1c
main.main.func1(0xcbe7a0, 0xc000272620, 0xc001b4a900)
/root/services/http/main.go:92 +0x41b fp=0xc000c3bbf0 sp=0xc000c3bb80 pc=0x98e2eb
net/http.HandlerFunc.ServeHTTP(0xae3b90, 0xcbe7a0, 0xc000272620, 0xc001b4a900)
/usr/local/go/src/net/http/server.go:2007 +0x44 fp=0xc000c3bc18 sp=0xc000c3bbf0 pc=0x6b7054
net/http.serverHandler.ServeHTTP(0xc0002722a0, 0xcbe7a0, 0xc000272620, 0xc001b4a900)
/usr/local/go/src/net/http/server.go:2802 +0xa4 fp=0xc000c3bc48 sp=0xc000c3bc18 pc=0x6b9c84
net/http.(*conn).serve(0xc000db0280, 0xcc26a0, 0xc0018e8580)
/usr/local/go/src/net/http/server.go:1890 +0x875 fp=0xc000c3bfc8 sp=0xc000c3bc48 pc=0x6b6085
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc000c3bfd0 sp=0xc000c3bfc8 pc=0x45c171
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2928 +0x384
 
goroutine 1 [chan receive, 174 minutes]:
main.main()
/root/services/http/main.go:145 +0x6fb
 
goroutine 4 [syscall, 174 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x41
 
goroutine 5 [chan receive]:
github.com/go-redis/redis/internal/pool.(*ConnPool).reaper(0xc0001ee420, 0xdf8475800)
/go/pkg/mod/github.com/go-redis/redis@v6.15.9+incompatible/internal/pool/pool.go:449 +0xba
created by github.com/go-redis/redis/internal/pool.NewConnPool
/go/pkg/mod/github.com/go-redis/redis@v6.15.9+incompatible/internal/pool/pool.go:99 +0x1af
 
goroutine 22 [select]:
github.com/jackc/pgx/v4/pgxpool.(*Pool).backgroundHealthCheck(0xc000224140)
/go/pkg/mod/github.com/jackc/pgx/v4@v4.6.0/pgxpool/pool.go:298 +0xe7
created by github.com/jackc/pgx/v4/pgxpool.ConnectConfig
/go/pkg/mod/github.com/jackc/pgx/v4@v4.6.0/pgxpool/pool.go:183 +0x23c
 
goroutine 25 [IO wait]:
internal/poll.runtime_pollWait(0x7f7e01241df8, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000260898, 0x72, 0x0, 0x0, 0xab84f7)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000260880, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc000260880, 0xc000053d88, 0x6ba794, 0xc000272340)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc000223900, 0x6184658f, 0xc000053d88, 0x4d79d6)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc000223900, 0xc000053dd8, 0x18, 0xc000073800, 0x6ba144)
/usr/local/go/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0002722a0, 0xcbe520, 0xc000223900, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2896 +0x280
net/http.(*Server).ListenAndServe(0xc0002722a0, 0x4, 0x7)
/usr/local/go/src/net/http/server.go:2825 +0xb7
main.main.func2(0xc0002722a0)
/root/services/http/main.go:138 +0x2f
created by main.main
/root/services/http/main.go:137 +0x5d1
 
goroutine 72942 [running]:
goroutine running on other thread; stack unavailable
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2928 +0x384
 
goroutine 72954 [IO wait]:
internal/poll.runtime_pollWait(0x7f7e00bb6ba8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001160998, 0x72, 0x0, 0x1, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001160980, 0xc001a823d1, 0x1, 0x1, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001160980, 0xc001a823d1, 0x1, 0x1, 0xc0018e84d8, 0xc000ba5f68, 0x47c4ec)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000226030, 0xc001a823d1, 0x1, 0x1, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).backgroundRead(0xc001a823c0)
/usr/local/go/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
/usr/local/go/src/net/http/server.go:673 +0xd4
 
goroutine 72943 [IO wait]:
internal/poll.runtime_pollWait(0x7f7e00bb6ad8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001160a18, 0x72, 0x0, 0x1, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001160a00, 0xc001129121, 0x1, 0x1, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001160a00, 0xc001129121, 0x1, 0x1, 0xc001b33318, 0xc00148f768, 0x47c4ec)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000226048, 0xc001129121, 0x1, 0x1, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).backgroundRead(0xc001129110)
/usr/local/go/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
/usr/local/go/src/net/http/server.go:673 +0xd4
 
goroutine 72942 [running]:
runtime.throw(0xabf0a8, 0x15)
/usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc000c37990 sp=0xc000c37960 pc=0x42f312
runtime.mapassign_faststr(0x9fa100, 0xc000263650, 0xc001d40360, 0x14, 0xc05937440e1a731f)
/usr/local/go/src/runtime/map_faststr.go:291 +0x3fe fp=0xc000c379f8 sp=0xc000c37990 pc=0x41495e
openreplay/backend/pkg/db/cache.(*PGCache).GetProjectByKey(0xc000263680, 0xc001d40360, 0x14, 0x0, 0x0, 0xa4f000)
/root/pkg/db/cache/project.go:17 +0x176 fp=0xc000c37a68 sp=0xc000c379f8 pc=0x957776
main.startSessionHandlerWeb(0xcbe7a0, 0xc0018f02a0, 0xc001132c00)
/root/services/http/handlers.go:59 +0x1cc fp=0xc000c37b80 sp=0xc000c37a68 pc=0x98bb1c
main.main.func1(0xcbe7a0, 0xc0018f02a0, 0xc001132c00)
/root/services/http/main.go:92 +0x41b fp=0xc000c37bf0 sp=0xc000c37b80 pc=0x98e2eb
net/http.HandlerFunc.ServeHTTP(0xae3b90, 0xcbe7a0, 0xc0018f02a0, 0xc001132c00)
/usr/local/go/src/net/http/server.go:2007 +0x44 fp=0xc000c37c18 sp=0xc000c37bf0 pc=0x6b7054
net/http.serverHandler.ServeHTTP(0xc0002722a0, 0xcbe7a0, 0xc0018f02a0, 0xc001132c00)
/usr/local/go/src/net/http/server.go:2802 +0xa4 fp=0xc000c37c48 sp=0xc000c37c18 pc=0x6b9c84
net/http.(*conn).serve(0xc000db0780, 0xcc26a0, 0xc001b33500)
/usr/local/go/src/net/http/server.go:1890 +0x875 fp=0xc000c37fc8 sp=0xc000c37c48 pc=0x6b6085
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc000c37fd0 sp=0xc000c37fc8 pc=0x45c171
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2928 +0x384
2021/11/04 22:58:24 /root/pkg/env/worker_id.go:18: Warning: unable to retrieve Fargate Task IP: Get : unsupported protocol scheme ""; trying to use HOSTNAME instead
2021/11/04 22:58:24 /root/services/http/main.go:142: Server successfully started on port 80

Any idea how to fix that?
Thank you

[estradino]

Fixed in v1.3.6.