goroutine 7059 [running]: runtime/pprof.writeGoroutineStacks(0x1050240, 0xc00054c0e0, 0x30, 0xe24b80) /home/dehort/dev/go/distros/go1.16/src/runtime/pprof/pprof.go:693 +0x9f runtime/pprof.writeGoroutine(0x1050240, 0xc00054c0e0, 0x2, 0xc0007b2000, 0x0) /home/dehort/dev/go/distros/go1.16/src/runtime/pprof/pprof.go:682 +0x45 runtime/pprof.(*Profile).WriteTo(0x14d3360, 0x1050240, 0xc00054c0e0, 0x2, 0xc00054c0e0, 0xc0002e01a0) /home/dehort/dev/go/distros/go1.16/src/runtime/pprof/pprof.go:331 +0x3f2 net/http/pprof.handler.ServeHTTP(0xc0005a2011, 0x9, 0x105c398, 0xc00054c0e0, 0xc0003e4800) /home/dehort/dev/go/distros/go1.16/src/net/http/pprof/pprof.go:253 +0x385 net/http/pprof.Index(0x105c398, 0xc00054c0e0, 0xc0003e4800) /home/dehort/dev/go/distros/go1.16/src/net/http/pprof/pprof.go:371 +0x8d7 net/http.HandlerFunc.ServeHTTP(0xefe590, 0x105c398, 0xc00054c0e0, 0xc0003e4800) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:2069 +0x44 net/http.(*ServeMux).ServeHTTP(0x152ba40, 0x105c398, 0xc00054c0e0, 0xc0003e4800) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:2448 +0x1ad github.com/redhatinsights/platform-go-middlewares/request_id.ConfiguredRequestID.func1.1(0x105c398, 0xc00054c0e0, 0xc0003e4700) /home/dehort/dev/go/pkg/mod/github.com/redhatinsights/platform-go-middlewares@v0.7.0/request_id/request_id.go:80 +0x315 net/http.HandlerFunc.ServeHTTP(0xc00062a270, 0x105c398, 0xc00054c0e0, 0xc0003e4700) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:2069 +0x44 github.com/gorilla/mux.(*Router).ServeHTTP(0xc0004100c0, 0x105c398, 0xc00054c0e0, 0xc0003e4500) /home/dehort/dev/go/pkg/mod/github.com/gorilla/mux@v1.8.0/mux.go:210 +0xd3 net/http.serverHandler.ServeHTTP(0xc00054c000, 0x105c398, 0xc00054c0e0, 0xc0003e4500) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:2887 +0xa3 net/http.(*conn).serve(0xc0000b6140, 0x105f3b8, 0xc0003c4300) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:1952 +0x8cd created by net/http.(*Server).Serve /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:3013 +0x39b goroutine 1 [chan receive, 40 minutes]: main.startMqttConnectionHandler(0x7ffc8025d4d1, 0xe) /home/dehort/dev/go/src/github.com/RedHatInsights/cloud-connector/cmd/cloud-connector/mqtt_connection_handler.go:118 +0xaf1 main.NewRootCommand.func1(0xc000660500, 0xc00007e080, 0x0, 0x2) /home/dehort/dev/go/src/github.com/RedHatInsights/cloud-connector/cmd/cloud-connector/main.go:23 +0x36 github.com/spf13/cobra.(*Command).execute(0xc000660500, 0xc00007e060, 0x2, 0x2, 0xc000660500, 0xc00007e060) /home/dehort/dev/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:856 +0x2c2 github.com/spf13/cobra.(*Command).ExecuteC(0xc000660280, 0x0, 0xffffffff, 0xc00010e058) /home/dehort/dev/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960 +0x375 github.com/spf13/cobra.(*Command).Execute(...) /home/dehort/dev/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897 main.main() /home/dehort/dev/go/src/github.com/RedHatInsights/cloud-connector/cmd/cloud-connector/main.go:46 +0x2a goroutine 25 [select, 40 minutes]: database/sql.(*DB).connectionOpener(0xc0002a2b60, 0x105f310, 0xc0000321c0) /home/dehort/dev/go/distros/go1.16/src/database/sql/sql.go:1133 +0xb5 created by database/sql.OpenDB /home/dehort/dev/go/distros/go1.16/src/database/sql/sql.go:740 +0x12a goroutine 77 [select, 40 minutes]: database/sql.(*DB).connectionOpener(0xc0002a2a90, 0x105f310, 0xc0001382c0) /home/dehort/dev/go/distros/go1.16/src/database/sql/sql.go:1133 +0xb5 created by database/sql.OpenDB /home/dehort/dev/go/distros/go1.16/src/database/sql/sql.go:740 +0x12a goroutine 32 [IO wait]: internal/poll.runtime_pollWait(0x7fbddabb9b08, 0x72, 0x0) /home/dehort/dev/go/distros/go1.16/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc0003bc098, 0x72, 0x0, 0x0, 0xeb50ff) /home/dehort/dev/go/distros/go1.16/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /home/dehort/dev/go/distros/go1.16/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0xc0003bc080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /home/dehort/dev/go/distros/go1.16/src/internal/poll/fd_unix.go:401 +0x212 net.(*netFD).accept(0xc0003bc080, 0xa97282549154f7ab, 0x0, 0x0) /home/dehort/dev/go/distros/go1.16/src/net/fd_unix.go:172 +0x45 net.(*TCPListener).accept(0xc00021e000, 0x60a7dca8, 0xc0004cfc20, 0x49e646) /home/dehort/dev/go/distros/go1.16/src/net/tcpsock_posix.go:139 +0x32 net.(*TCPListener).Accept(0xc00021e000, 0xc0004cfc70, 0x18, 0xc000102a80, 0x73771b) /home/dehort/dev/go/distros/go1.16/src/net/tcpsock.go:261 +0x65 net/http.(*Server).Serve(0xc00054c000, 0x105c188, 0xc00021e000, 0x0, 0x0) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:2981 +0x285 net/http.(*Server).ListenAndServe(0xc00054c000, 0x4, 0xec1bd1) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:2910 +0xba github.com/RedHatInsights/cloud-connector/internal/platform/utils.StartHTTPServer.func1(0xeb7427, 0xa, 0x7ffc8025d4d1, 0xe, 0xc00054c000) /home/dehort/dev/go/src/github.com/RedHatInsights/cloud-connector/internal/platform/utils/utils.go:22 +0x10d created by github.com/RedHatInsights/cloud-connector/internal/platform/utils.StartHTTPServer /home/dehort/dev/go/src/github.com/RedHatInsights/cloud-connector/internal/platform/utils/utils.go:20 +0xb1 goroutine 82 [syscall, 40 minutes]: os/signal.signal_recv(0x0) /home/dehort/dev/go/distros/go1.16/src/runtime/sigqueue.go:168 +0xa5 os/signal.loop() /home/dehort/dev/go/distros/go1.16/src/os/signal/signal_unix.go:23 +0x25 created by os/signal.Notify.func1.1 /home/dehort/dev/go/distros/go1.16/src/os/signal/signal.go:151 +0x45 goroutine 5612 [semacquire, 12 minutes]: sync.runtime_Semacquire(0xc0003561f0) /home/dehort/dev/go/distros/go1.16/src/runtime/sema.go:56 +0x45 sync.(*WaitGroup).Wait(0xc0003561e8) /home/dehort/dev/go/distros/go1.16/src/sync/waitgroup.go:130 +0x65 github.com/eclipse/paho%2emqtt%2egolang.(*client).stopCommsWorkers.func1(0xc000356000, 0xc000672060) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:614 +0xa7 created by github.com/eclipse/paho%2emqtt%2egolang.(*client).stopCommsWorkers /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:612 +0x17c goroutine 7020 [chan send, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.ackFunc.func1() /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:458 +0x325 sync.(*Once).doSlow(0xc00072c0d8, 0xc0006d0120) /home/dehort/dev/go/distros/go1.16/src/sync/once.go:68 +0xec sync.(*Once).Do(...) /home/dehort/dev/go/distros/go1.16/src/sync/once.go:59 github.com/eclipse/paho%2emqtt%2egolang.(*message).Ack(0xc00072c0a0) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/message.go:73 +0x49 github.com/eclipse/paho%2emqtt%2egolang.(*router).matchAndDispatch.func1(0xc0000fca80, 0xc000394ed0, 0xc0000fcae0, 0xc000356000, 0xe35501) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/router.go:174 +0x374 created by github.com/eclipse/paho%2emqtt%2egolang.(*router).matchAndDispatch /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/router.go:136 +0x88 goroutine 7043 [chan send, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.startComms.func2(0xc0000fcea0, 0xc0000fcf60, 0xc0003dfc80) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:425 +0x5f created by github.com/eclipse/paho%2emqtt%2egolang.startComms /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:423 +0x274 goroutine 7045 [chan send, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers.func2(0xc0006b6070, 0xc0006b6078, 0xc0000fca80, 0xc000356000) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:567 +0x245 created by github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:554 +0x4b3 goroutine 7044 [semacquire, 12 minutes]: sync.runtime_Semacquire(0xc0003dfc88) /home/dehort/dev/go/distros/go1.16/src/runtime/sema.go:56 +0x45 sync.(*WaitGroup).Wait(0xc0003dfc80) /home/dehort/dev/go/distros/go1.16/src/sync/waitgroup.go:130 +0x65 github.com/eclipse/paho%2emqtt%2egolang.startComms.func3(0xc0003dfc80, 0xc0000fcf60) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:432 +0x2f created by github.com/eclipse/paho%2emqtt%2egolang.startComms /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:431 +0x2a5 goroutine 7042 [chan send, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.startComms.func1(0xc0000fcde0, 0xc0000fcf60, 0xc0000fce40, 0xc0000fcf00, 0xc0003dfc80) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:411 +0xc5 created by github.com/eclipse/paho%2emqtt%2egolang.startComms /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:400 +0x23e goroutine 7025 [chan send, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.startOutgoingComms.func1(0xc0006b6090, 0xc0006b6098, 0xc0006b60a0, 0xc0000fcea0, 0x10660f0, 0xc000356000, 0x1066148, 0xc00037cc90) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:321 +0x935 created by github.com/eclipse/paho%2emqtt%2egolang.startOutgoingComms /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/net.go:259 +0x1bf goroutine 5613 [chan receive, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.(*client).internalConnLost.func1(0xc000672060, 0xc000356000, 0x104ee80, 0xc0006f28b0) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:460 +0xa7 created by github.com/eclipse/paho%2emqtt%2egolang.(*client).internalConnLost /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:458 +0xed goroutine 7049 [IO wait]: internal/poll.runtime_pollWait(0x7fbddabb9a20, 0x72, 0xffffffffffffffff) /home/dehort/dev/go/distros/go1.16/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc00018c018, 0x72, 0x0, 0x1, 0xffffffffffffffff) /home/dehort/dev/go/distros/go1.16/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /home/dehort/dev/go/distros/go1.16/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc00018c000, 0xc00062a101, 0x1, 0x1, 0x0, 0x0, 0x0) /home/dehort/dev/go/distros/go1.16/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc00018c000, 0xc00062a101, 0x1, 0x1, 0xc0004e1ec0, 0x0, 0xc000138298) /home/dehort/dev/go/distros/go1.16/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc0003a2000, 0xc00062a101, 0x1, 0x1, 0x0, 0x0, 0x0) /home/dehort/dev/go/distros/go1.16/src/net/net.go:183 +0x91 net/http.(*connReader).backgroundRead(0xc00062a0f0) /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:692 +0x58 created by net/http.(*connReader).startBackgroundRead /home/dehort/dev/go/distros/go1.16/src/net/http/server.go:688 +0xd5 goroutine 7022 [chan send, 12 minutes]: github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers.func1(0xc000356000, 0xc0000fcc60, 0xc0000fcb40, 0xc0006b6068) /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:540 +0x2a5 created by github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers /home/dehort/dev/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.3.1/client.go:521 +0x37f