Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fatal error: concurrent map writes #3387

Open
zouzehe opened this issue Nov 4, 2022 · 3 comments
Open

fatal error: concurrent map writes #3387

zouzehe opened this issue Nov 4, 2022 · 3 comments

Comments

@zouzehe
Copy link

zouzehe commented Nov 4, 2022

Description

Very randomly, I got my Gin application fatal error

How to reproduce

func (api ChaptersApi) Report(ctx *gin.Context) {
	records, body, err := tool.InputDataDecode(ctx, tool.AppChapters, api.log)
	if err != nil {
		api.log.Errorf("Chapters Response Body Error: InputDataDecode Error: %+v. body(%+v)", err, body)
		response.Response(ctx, err.Error())
		return
	}

	_, err = api.uc.ChaptersReport(ctx, records)
	if err != nil {
		header := ctx.Request.Header
		msg := fmt.Sprintf("Chapters Response Body Error: %+v,%+v", err, header)
		api.log.Errorf(msg)
		response.Response(ctx, err.Error())
		return
	}
	response.Response(ctx, "ok")
}

// Response
func Response(c *gin.Context, msg string) {
	response := GetResponseBody(msg)
	payload, _ := json.Marshal(response)
	platform := tool.GetHeaderByName(c, "platform")
	respMsg := ""
	if platform == "ios" {
		respMsg, _ = tool.EncodeB64ForIos(string(payload))
	} else {
		respMsg, _ = tool.EncodeB64ForAndroid(string(payload))
	}

	c.Data(response.HttpStatus, "text/html; charset=utf-8", []byte(respMsg))
}

Expectations

no fatal error

Actual result

fatal error: concurrent map writes

goroutine 4906802 [running]:
runtime.throw(0x1496b9b, 0x15)
	/usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc00087dbd8 sp=0xc00087dba8 pc=0x43a272
runtime.mapassign_faststr(0x13f9900, 0xc0001f8a20, 0x148cf9a, 0xc, 0x1f2bc40)
	/usr/local/go/src/runtime/map_faststr.go:291 +0x3d8 fp=0xc00087dc40 sp=0xc00087dbd8 pc=0x416518
github.com/gin-gonic/gin/render.writeContentType(0x7f7ac39194a0, 0xc0006a6b00, 0xc00077a110, 0x1, 0x1)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/render/render.go:38 +0xaa fp=0xc00087dc80 sp=0xc00087dc40 pc=0xcac26a
github.com/gin-gonic/gin/render.Data.WriteContentType(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/render/data.go:24
github.com/gin-gonic/gin/render.Data.Render(0x149a80c, 0x18, 0xc0007aa440, 0x38, 0x40, 0x7f7ac39194a0, 0xc0006a6b00, 0x28, 0xc00087dd60)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/render/data.go:17 +0xaa fp=0xc00087dcf0 sp=0xc00087dc80 pc=0xca9e0a
github.com/gin-gonic/gin/render.(*Data).Render(0xc0001f8270, 0x7f7ac39194a0, 0xc0006a6b00, 0x7f7ac39194a0, 0xc0006a6b00)
	<autogenerated>:1 +0x91 fp=0xc00087dd70 sp=0xc00087dcf0 pc=0xcacdf1
github.com/gin-gonic/gin.(*Context).Render(0xc0006a6b00, 0xc8, 0x16569b8, 0xc0001f8270)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:913 +0x149 fp=0xc00087ddb0 sp=0xc00087dd70 pc=0xcb3349
github.com/gin-gonic/gin.(*Context).Data(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:1003
	/data/web/internal/pkg/response/response.go:58 +0x205 fp=0xc00087de80 sp=0xc00087ddb0 pc=0xcd0965
	/data/web/internal/nexus/api/v1/chapters.go:42 +0x38f fp=0xc00087df68 sp=0xc00087de80 pc=0xcd0e2f
	/data/web/internal/nexus/api/v1/chapters.go:24 +0x46 fp=0xc00087df90 sp=0xc00087df68 pc=0xcd27a6
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1.1(0xc0035908a0, 0xc0006a6b00, 0xc00002a600)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:61 +0x68 fp=0xc00087dfc8 sp=0xc00087df90 pc=0x119d988
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc00087dfd0 sp=0xc00087dfc8 pc=0x4744a1
created by gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:55 +0x334

goroutine 1 [semacquire, 39 minutes]:
sync.runtime_Semacquire(0xc0000bdbd0)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc0000bdbc8)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc0000bdbc0, 0xc00029da10, 0x3)
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:53 +0x31
gitlab.stardustgod.com/infra/public-core-component/pkg/app.(*App).Run(0xc0001ef170, 0x16446c0, 0xc0005e00a0)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/app/app.go:84 +0x369
main.main()
	/data/web/cmd/nexus-server/main.go:131 +0x2c5

goroutine 26 [chan receive, 39 minutes]:
github.com/nacos-group/nacos-sdk-go/common/security.(*AuthClient).AutoRefresh.func1(0xc000542480)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/security/security_proxy.go:81 +0x8b
created by github.com/nacos-group/nacos-sdk-go/common/security.(*AuthClient).AutoRefresh
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/security/security_proxy.go:76 +0x50

goroutine 84 [chan receive, 39 minutes]:
gitlab.stardustgod.com/infra/public-core-component/pkg/config/nacos.(*nacos).Watch.func2(0xc000266880, 0xc0005e2000, 0xc0005e2060)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/config/nacos/config.go:245 +0x88
created by gitlab.stardustgod.com/infra/public-core-component/pkg/config/nacos.(*nacos).Watch
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/config/nacos/config.go:243 +0x1ce

goroutine 86 [chan receive]:
github.com/alibaba/sentinel-golang/core/log/metric.writeTaskLoop()
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/log/metric/aggregator.go:80 +0x2a6
github.com/alibaba/sentinel-golang/util.RunWithRecover(0x14f84a0)
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/auto_recover.go:28 +0x49
created by github.com/alibaba/sentinel-golang/core/log/metric.InitTask.func1
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/log/metric/aggregator.go:59 +0xcf

goroutine 85 [chan receive, 39 minutes]:
main.loadConfig.func1(0x165c678, 0xc000563220, 0xc000010650, 0xc0005e2000)
	/data/web/cmd/nexus-server/main.go:71 +0x38
created by main.loadConfig
	/data/web/cmd/nexus-server/main.go:69 +0x554

goroutine 145 [IO wait]:
internal/poll.runtime_pollWait(0x7f7ac426a860, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0003a1d18, 0x72, 0x0, 0x0, 0x1487ab1)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0003a1d00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc0003a1d00, 0x203000, 0xc0005f2ca0, 0x18)
	/usr/local/go/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc0003a57d0, 0x0, 0xc0005f2c70, 0x474607)
	/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).AcceptTCP(0xc0003a57d0, 0x0, 0xc0005f2c98, 0x47173b)
	/usr/local/go/src/net/tcpsock.go:248 +0x65
github.com/fvbock/endless.(*endlessListener).Accept(0xc0005fbe60, 0xc0005f2d10, 0x18, 0xc000001b00, 0x72249b)
	/go/pkg/mod/github.com/fvbock/endless@v0.0.0-20170109170031-447134032cb6/endless.go:490 +0x46
net/http.(*Server).Serve(0xc0003b4b40, 0x165baa8, 0xc0005fbe60, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2961 +0x285
github.com/fvbock/endless.(*endlessServer).Serve(0xc0003b4b40, 0x0, 0x0)
	/go/pkg/mod/github.com/fvbock/endless@v0.0.0-20170109170031-447134032cb6/endless.go:195 +0x128
github.com/fvbock/endless.(*endlessServer).ListenAndServe(0xc0003b4b40, 0xffffffffffffffff, 0x0)
	/go/pkg/mod/github.com/fvbock/endless@v0.0.0-20170109170031-447134032cb6/endless.go:229 +0x1b7
gitlab.stardustgod.com/infra/public-core-component/pkg/app.(*App).Run.func2(0x0, 0x0)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/app/app.go:67 +0x4b
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0000bdbc0, 0xc0003a57b8)
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:75 +0x53
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:72 +0x71

goroutine 48 [chan receive]:
github.com/nacos-group/nacos-sdk-go/clients/config_client.(*ConfigClient).delayScheduler(0xc0002684e0, 0xc00024acd0, 0x989680, 0x1484985, 0x4, 0xc000563290)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/clients/config_client/config_client.go:355 +0x3d
created by github.com/nacos-group/nacos-sdk-go/clients/config_client.NewConfigClient
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/clients/config_client/config_client.go:82 +0x36a

goroutine 213 [runnable]:
github.com/segmentio/kafka-go/protocol.makeArray(0x167b5a8, 0x12600a0, 0x2, 0x4, 0xc0002bd810, 0x4a555c)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/reflect.go:86 +0x89
github.com/segmentio/kafka-go/protocol.(*decoder).decodeArray(0xc002cd2a80, 0x1248fe0, 0xc000272468, 0x197, 0x167b5a8, 0x12600a0, 0x14f91f0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:108 +0x66
github.com/segmentio/kafka-go/protocol.arrayDecodeFuncOf.func2(0xc002cd2a80, 0x1248fe0, 0xc000272468, 0x197)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:441 +0x65
github.com/segmentio/kafka-go/protocol.structDecodeFuncOf.func2(0xc002cd2a80, 0x13d2000, 0xc000272458, 0x199)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:410 +0x105
github.com/segmentio/kafka-go/protocol.(*decoder).decodeArray(0xc002cd2a80, 0x1246b20, 0xc0001d1120, 0x197, 0x167b5a8, 0x13d2000, 0xc0004a7d60)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:110 +0xe6
github.com/segmentio/kafka-go/protocol.arrayDecodeFuncOf.func2(0xc002cd2a80, 0x1246b20, 0xc0001d1120, 0x197)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:441 +0x65
github.com/segmentio/kafka-go/protocol.structDecodeFuncOf.func2(0xc002cd2a80, 0x13b1b80, 0xc0001d1100, 0x199)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:410 +0x105
github.com/segmentio/kafka-go/protocol.(*decoder).decodeArray(0xc002cd2a80, 0x1246b60, 0xc001607598, 0x197, 0x167b5a8, 0x13b1b80, 0xc0004a7da0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:110 +0xe6
github.com/segmentio/kafka-go/protocol.arrayDecodeFuncOf.func2(0xc002cd2a80, 0x1246b60, 0xc001607598, 0x197)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:441 +0x65
github.com/segmentio/kafka-go/protocol.structDecodeFuncOf.func2(0xc002cd2a80, 0x13c1f80, 0xc001607560, 0x199)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/decode.go:410 +0x105
github.com/segmentio/kafka-go/protocol.ReadResponse(0x7f7ac3f7cf60, 0xc0000bca80, 0xc000070003, 0x7f7ac3f7cf60, 0xc0000bca80, 0x1645900, 0xc0000d42a0, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/response.go:76 +0x4d2
github.com/segmentio/kafka-go/protocol.RoundTrip(0x1655900, 0xc0000bca80, 0x30d000d0007, 0x0, 0x0, 0x16459c0, 0xc0000d42a0, 0x5757a9, 0xc0000aa580, 0xc0d10f775688f6fd, ...)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/roundtrip.go:15 +0x186
github.com/segmentio/kafka-go/protocol.(*Conn).RoundTrip(0xc0000bca80, 0x16459c0, 0xc0000d42a0, 0x1ef8520, 0x0, 0x0, 0xc000f25010)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/protocol/conn.go:94 +0x1c5
github.com/segmentio/kafka-go.(*conn).roundTrip(0xc00024aaa0, 0x1661860, 0xc0016074a0, 0xc0000bca80, 0x16459c0, 0xc0000d42a0, 0x0, 0x0, 0x0, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1271 +0xfe
github.com/segmentio/kafka-go.(*conn).run(0xc00024aaa0, 0xc0000bca80, 0xc00010a8a0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1247 +0x14b
created by github.com/segmentio/kafka-go.(*connGroup).connect
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1224 +0x9a5

goroutine 69 [IO wait]:
internal/poll.runtime_pollWait(0x7f7ac426aeb8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc00056e498, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00056e480, 0xc0005e8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc00056e480, 0xc0005e8000, 0x1000, 0x1000, 0xc0005f3e88, 0xc0005f3d18, 0x2)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000560290, 0xc0005e8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
net/http.(*persistConn).Read(0xc0002b07e0, 0xc0005e8000, 0x1000, 0x1000, 0xc0005f3dc4, 0x2, 0x2)
	/usr/local/go/src/net/http/transport.go:1922 +0x77
bufio.(*Reader).fill(0xc0005de060)
	/usr/local/go/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).Peek(0xc0005de060, 0x1, 0x0, 0x1, 0x1, 0x1, 0x0)
	/usr/local/go/src/bufio/bufio.go:139 +0x4f
net/http.(*persistConn).readLoop(0xc0002b07e0)
	/usr/local/go/src/net/http/transport.go:2083 +0x1a8
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1743 +0xc77

goroutine 70 [select]:
net/http.(*persistConn).writeLoop(0xc0002b07e0)
	/usr/local/go/src/net/http/transport.go:2382 +0xf7
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1744 +0xc9c

goroutine 87 [select]:
github.com/alibaba/sentinel-golang/core/log/metric.InitTask.func1.1()
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/log/metric/aggregator.go:64 +0x9c
github.com/alibaba/sentinel-golang/util.RunWithRecover(0xc0005e67b0)
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/auto_recover.go:28 +0x49
created by github.com/alibaba/sentinel-golang/core/log/metric.InitTask.func1
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/log/metric/aggregator.go:62 +0x150

goroutine 88 [select]:
github.com/alibaba/sentinel-golang/core/system_metric.InitLoadCollector.func1.1()
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/system_metric/sys_metric_stat.go:212 +0x9c
github.com/alibaba/sentinel-golang/util.RunWithRecover(0xc0005e67e0)
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/auto_recover.go:28 +0x49
created by github.com/alibaba/sentinel-golang/core/system_metric.InitLoadCollector.func1
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/system_metric/sys_metric_stat.go:210 +0xb0

goroutine 89 [select]:
github.com/alibaba/sentinel-golang/core/system_metric.InitCpuCollector.func1.1()
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/system_metric/sys_metric_stat.go:160 +0x9c
github.com/alibaba/sentinel-golang/util.RunWithRecover(0xc0005e6930)
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/auto_recover.go:28 +0x49
created by github.com/alibaba/sentinel-golang/core/system_metric.InitCpuCollector.func1
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/system_metric/sys_metric_stat.go:158 +0xb0

goroutine 90 [select]:
github.com/alibaba/sentinel-golang/core/system_metric.InitMemoryCollector.func1.1()
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/system_metric/sys_metric_stat.go:102 +0x9c
github.com/alibaba/sentinel-golang/util.RunWithRecover(0xc0005e6960)
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/auto_recover.go:28 +0x49
created by github.com/alibaba/sentinel-golang/core/system_metric.InitMemoryCollector.func1
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/core/system_metric/sys_metric_stat.go:100 +0xb0

goroutine 91 [sleep]:
time.Sleep(0xf4240)
	/usr/local/go/src/runtime/time.go:193 +0xd2
github.com/alibaba/sentinel-golang/util.StartTimeTicker.func1()
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/time_ticker.go:32 +0x6c
created by github.com/alibaba/sentinel-golang/util.StartTimeTicker
	/go/pkg/mod/github.com/alibaba/sentinel-golang@v1.0.4/util/time_ticker.go:28 +0xa5

goroutine 97 [select]:
github.com/go-redis/redis/v8/internal/pool.(*ConnPool).reaper(0xc0005f40a0, 0xdf8475800)
	/go/pkg/mod/github.com/go-redis/redis/v8@v8.10.0/internal/pool/pool.go:455 +0xd7
created by github.com/go-redis/redis/v8/internal/pool.NewConnPool
	/go/pkg/mod/github.com/go-redis/redis/v8@v8.10.0/internal/pool/pool.go:110 +0x1fc

goroutine 98 [select, 39 minutes]:
database/sql.(*DB).connectionOpener(0xc0004f08f0, 0x16617f0, 0xc0000bc980)
	/usr/local/go/src/database/sql/sql.go:1133 +0xb5
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:740 +0x12a

goroutine 75 [select, 39 minutes]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc000418300, 0xc00040cb40, 0xc000216240)
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/connection.go:614 +0xa5
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/connection.go:611 +0xbe

goroutine 30 [select]:
net/http.(*persistConn).roundTrip(0xc0002b07e0, 0xc000fea0c0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:2610 +0x765
net/http.(*Transport).roundTrip(0x1ee9860, 0xc0006a6500, 0x143ec20, 0x201, 0xc0006a6500)
	/usr/local/go/src/net/http/transport.go:592 +0xacb
net/http.(*Transport).RoundTrip(0x1ee9860, 0xc0006a6500, 0x1ee9860, 0xc0d10f77e5435956, 0x22d71d7c2ae)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc0006a6400, 0x1646740, 0x1ee9860, 0xc0d10f77e5435956, 0x22d71d7c2ae, 0x1ef8520, 0xc000208158, 0xc0d10f77e5435956, 0x1, 0x0)
	/usr/local/go/src/net/http/client.go:251 +0x454
net/http.(*Client).send(0xc001624480, 0xc0006a6400, 0xc0d10f77e5435956, 0x22d71d7c2ae, 0x1ef8520, 0xc000208158, 0x0, 0x1, 0xc001624510)
	/usr/local/go/src/net/http/client.go:175 +0xff
net/http.(*Client).do(0xc001624480, 0xc0006a6400, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:717 +0x45f
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:585
github.com/nacos-group/nacos-sdk-go/common/http_agent.post(0xc00009a0a0, 0x49, 0xc001624450, 0x80e8, 0xc001624300, 0x12c7dc0, 0xc001624300, 0x148c529)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/http_agent/post.go:37 +0x15d
github.com/nacos-group/nacos-sdk-go/common/http_agent.(*HttpAgent).Post(...)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/http_agent/http_agent.go:95
github.com/nacos-group/nacos-sdk-go/common/http_agent.(*HttpAgent).Request(0x1f29da0, 0x14842fd, 0x4, 0xc00009a0a0, 0x49, 0xc001624450, 0x80e8, 0xc001624300, 0x49, 0xc000497550, ...)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/http_agent/http_agent.go:79 +0x445
github.com/nacos-group/nacos-sdk-go/common/nacos_server.(*NacosServer).callConfigServer(0xc000137600, 0x14986bf, 0x17, 0xc001624300, 0xc001624330, 0x14842fd, 0x4, 0xc0003c2000, 0x2c, 0x1485e1c, ...)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/nacos_server/nacos_server.go:116 +0xaa7
github.com/nacos-group/nacos-sdk-go/common/nacos_server.(*NacosServer).ReqConfigApi(0xc000137600, 0x14986bf, 0x17, 0xc001624300, 0xc001624330, 0x14842fd, 0x4, 0x80e8, 0xb5000000013fa080, 0x1, ...)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/common/nacos_server/nacos_server.go:190 +0x205
github.com/nacos-group/nacos-sdk-go/clients/config_client.(*ConfigProxy).ListenConfig(0xc000268518, 0xc001624300, 0x1491e00, 0xc000048090, 0x24, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/clients/config_client/config_proxy.go:184 +0x2aa
github.com/nacos-group/nacos-sdk-go/clients/config_client.(*ConfigClient).longPulling.func1(0xc00007f7a0, 0x0)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/clients/config_client/config_client.go:421 +0x511
github.com/nacos-group/nacos-sdk-go/clients/config_client.(*ConfigClient).delayScheduler(0xc0002684e0, 0xc00024a0f0, 0x989680, 0x14a98d2, 0x1, 0xc00022e030)
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/clients/config_client/config_client.go:357 +0x47
created by github.com/nacos-group/nacos-sdk-go/clients/config_client.(*ConfigClient).listenConfigExecutor.func1
	/go/pkg/mod/github.com/nacos-group/nacos-sdk-go@v1.0.9/clients/config_client/config_client.go:373 +0x1cb

goroutine 2423 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0039bab10, 0x318)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0039bab00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000f25b78, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000f25b60)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc002871f00)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 178 [chan receive, 39 minutes]:
github.com/fvbock/endless.(*endlessServer).handleSignals(0xc0003b4b40)
	/go/pkg/mod/github.com/fvbock/endless@v0.0.0-20170109170031-447134032cb6/endless.go:326 +0xef
created by github.com/fvbock/endless.(*endlessServer).ListenAndServe
	/go/pkg/mod/github.com/fvbock/endless@v0.0.0-20170109170031-447134032cb6/endless.go:213 +0x54

goroutine 144 [chan receive, 39 minutes]:
gitlab.stardustgod.com/infra/public-core-component/pkg/app.(*App).Run.func1(0x0, 0x0)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/app/app.go:60 +0x5d
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0000bdbc0, 0xc0005fbe40)
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:75 +0x53
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:72 +0x71

goroutine 4906842 [select]:
github.com/segmentio/kafka-go.(*partitionWriter).awaitBatch(0xc000207380, 0xc0003ecbd0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1044 +0x76
github.com/segmentio/kafka-go.(*partitionWriter).newWriteBatch.func1()
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1036 +0x33
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc600, 0xc0005e6af8)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 58 [select, 39 minutes]:
gitlab.stardustgod.com/infra/public-core-component/pkg/app.(*App).Run.func3(0x0, 0x0)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/app/app.go:76 +0xb5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0000bdbc0, 0xc00029da10)
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:75 +0x53
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/sync@v0.0.0-20220722155255-886fb9371eb4/errgroup/errgroup.go:72 +0x71

goroutine 57 [syscall, 39 minutes]:
os/signal.signal_recv(0x4744a6)
	/usr/local/go/src/runtime/sigqueue.go:168 +0xa5
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x45

goroutine 2658 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000feba10, 0x2f4c)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000feba00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc001960678, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc001960660)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc300, 0xc0008c1180)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906809 [select]:
github.com/segmentio/kafka-go.(*Writer).WriteMessages(0xc0001dc100, 0x1661828, 0xc000044078, 0xc00055ae10, 0x1, 0x1, 0x0, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:659 +0x96b
	/data/web/internal/nexus/data/data_transform.go:254 +0x16a5
	/data/web/internal/nexus/biz/common.go:36
	/data/web/internal/nexus/api/v1/chapters.go:32 +0x1f1
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1.1(0xc00496a720, 0xc0006a6c00, 0xc0005e2720)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:61 +0x68
created by gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:55 +0x334

goroutine 227 [select]:
github.com/segmentio/kafka-go.async.await(0xc001607200, 0x1661860, 0xc0016074a0, 0x1661860, 0xc0016074a0, 0xc00077ad60, 0x2)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:873 +0xb3
github.com/segmentio/kafka-go.(*connPool).discover(0xc0000fe240, 0x16617f0, 0xc00055cd40, 0xc0002441e0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:606 +0x533
created by github.com/segmentio/kafka-go.(*Transport).grabPool
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:254 +0x4cd

goroutine 1533 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc001e76890, 0x628)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc001e76880)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc001c93638, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc001c93620)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc600, 0xc002282020)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 6723 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000e3b3d0, 0x31c)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000e3b3c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc001145398, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc001145380)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc0032bfde0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 239 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055d9d0, 0x13e2d)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055d9c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005df6f8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005df6e0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc0002186d0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906791 [IO wait]:
internal/poll.runtime_pollWait(0x7f7ac4004c28, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0003a0098, 0x72, 0x0, 0x1, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0003a0080, 0xc0039ec2b1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc0003a0080, 0xc0039ec2b1, 0x1, 0x1, 0x4, 0xc00048cf08, 0x44e9df)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0002081c8, 0xc0039ec2b1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
net/http.(*connReader).backgroundRead(0xc0039ec2a0)
	/usr/local/go/src/net/http/server.go:672 +0x58
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:668 +0xd5

goroutine 241 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055da10, 0x13e0d)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055da00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005df7b8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005df7a0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000218890)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 243 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055da50, 0xc000013e3e)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055da40)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005df878, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005df860)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc0002188c0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906792 [select]:
gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1(0xc0009cc100)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:66 +0x3df
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
	/data/web/internal/pkg/middleware/sentinel.go:105 +0x122
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
	/data/web/internal/pkg/middleware/logger.go:45 +0xb8
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
	/data/web/internal/pkg/middleware/recovery.go:50 +0x70
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc00040e1a0, 0xc0009cc100)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/gin.go:489 +0x2aa
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc00040e1a0, 0x165cbb8, 0xc0006de0e0, 0xc0009cc000)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/gin.go:445 +0x15c
net/http.serverHandler.ServeHTTP(0xc0003b4b40, 0x165cbb8, 0xc0006de0e0, 0xc0009cc000)
	/usr/local/go/src/net/http/server.go:2867 +0xa3
net/http.(*conn).serve(0xc000012c80, 0x1661898, 0xc002cd2040)
	/usr/local/go/src/net/http/server.go:1932 +0x8cd
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2993 +0x39b

goroutine 245 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055da90, 0x13ed0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055da80)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005df938, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005df920)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000218910)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 247 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055dad0, 0x13e21)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055dac0)
	/usr/local/go/src/sync/cond.go:56 +0x99
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906777 [select]:
github.com/segmentio/kafka-go.(*Writer).WriteMessages(0xc0001dc100, 0x1661828, 0xc000044078, 0xc00052e120, 0x1, 0x1, 0x0, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:659 +0x96b
	/data/web/internal/nexus/data/data_transform.go:254 +0x16a5
	/data/web/internal/nexus/biz/common.go:36
	/data/web/internal/nexus/api/v1/chapters.go:32 +0x1f1
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1.1(0xc000f24a80, 0xc0009ccf00, 0xc000244540)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:61 +0x68
created by gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:55 +0x334

goroutine 4906837 [IO wait]:
internal/poll.runtime_pollWait(0x7f7ac38d52a0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000168518, 0x72, 0x0, 0x1, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000168500, 0xc0001df6c1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000168500, 0xc0001df6c1, 0x1, 0x1, 0xc0004bcca0, 0x16565f0, 0xc0005c7770)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000560280, 0xc0001df6c1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
net/http.(*connReader).backgroundRead(0xc0001df6b0)
	/usr/local/go/src/net/http/server.go:672 +0x58
created by net/http.(*connReader).startBackgroundRead
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005df9f8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005df9e0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000218b30)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 249 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055dd90, 0x13e8d)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055dd80)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000206018, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000206000)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000218c70)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 251 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055ddd0, 0x13e23)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055ddc0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0002060d8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0002060c0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000218c90)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 3240 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00047f9d0, 0x2f1c)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00047f9c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc001960df8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc001960de0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc300, 0xc00133bbc0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 253 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00055de10, 0x13e8b)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00055de00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000206918, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000206900)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000218cb0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 1324563 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc003d0ab10, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
	/usr/local/go/src/net/http/server.go:668 +0xd5

goroutine 641 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000b390d0, 0x608)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000b390c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000c3d1b8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000c3d1a0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc600, 0xc00073ba20)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906840 [IO wait]:
internal/poll.runtime_pollWait(0x7f7ac389ae88, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000168618, 0x72, 0x0, 0x1, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000168600, 0xc0001dfc31, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000168600, 0xc0001dfc31, 0x1, 0x1, 0xc002cfaf60, 0x16565f0, 0xc0005c4f70)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000560338, 0xc0001dfc31, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
net/http.(*connReader).backgroundRead(0xc0001dfc20)
	/usr/local/go/src/net/http/server.go:672 +0x58
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:668 +0xd5

goroutine 3297 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0007af710, 0x1087)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0007af700)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc002cd11b8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc002cd11a0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc400, 0xc00135fb20)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 727 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000223cd0, 0x62d)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000223cc0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000207398, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000207380)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc600, 0xc0005ffa90)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 3647 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004a6e450, 0x319)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc004a6e440)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc004b82678, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc004b82660)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc004aaf050)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 13100 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc002b72450, 0x317)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc002b72440)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc001c463d8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc001c463c0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc0004522e0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906835 [select]:
gitlab.stardustgod.com/infra/public-core-component/pkg/timeout/gin.Timeout.func1(0xc0006a7000)
	/go/pkg/mod/gitlab.stardustgod.com/infra/public-core-component@v0.0.0-20220407095715-36316d365e74/pkg/timeout/gin/timeout.go:66 +0x3df
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
	/data/web/internal/pkg/middleware/sentinel.go:105 +0x122
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
	/data/web/internal/pkg/middleware/logger.go:45 +0xb8
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
	/data/web/internal/pkg/middleware/recovery.go:50 +0x70
github.com/gin-gonic/gin.(*Context).Next(...)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/context.go:165
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc00040e1a0, 0xc0006a7000)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/gin.go:489 +0x2aa
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc00040e1a0, 0x165cbb8, 0xc0000ee460, 0xc0006a6f00)
	/go/pkg/mod/github.com/gin-gonic/gin@v1.7.2/gin.go:445 +0x15c
net/http.serverHandler.ServeHTTP(0xc0003b4b40, 0x165cbb8, 0xc0000ee460, 0xc0006a6f00)
	/usr/local/go/src/net/http/server.go:2867 +0xa3
net/http.(*conn).serve(0xc00058a280, 0x1661898, 0xc001e76400)
	/usr/local/go/src/net/http/server.go:1932 +0x8cd
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2993 +0x39b

goroutine 4906800 [select]:
github.com/segmentio/kafka-go.(*partitionWriter).awaitBatch(0xc000206000, 0xc000348620)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1044 +0x76
github.com/segmentio/kafka-go.(*partitionWriter).newWriteBatch.func1()
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1036 +0x33
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000020f78)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4505594 [chan receive]:
github.com/segmentio/kafka-go.(*conn).run(0xc000553a40, 0xc001f12700, 0xc0005e25a0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1246 +0xa5
created by github.com/segmentio/kafka-go.(*connGroup).connect
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1224 +0x9a5

goroutine 4906834 [select]:
github.com/segmentio/kafka-go.(*partitionWriter).awaitBatch(0xc000206d20, 0xc0003ec930)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1044 +0x76
github.com/segmentio/kafka-go.(*partitionWriter).newWriteBatch.func1()
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1036 +0x33
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc0005e6540)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906812 [select]:
github.com/segmentio/kafka-go.(*partitionWriter).awaitBatch(0xc000206ba0, 0xc0003ec5b0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1044 +0x76
github.com/segmentio/kafka-go.(*partitionWriter).newWriteBatch.func1()
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1036 +0x33
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc0000cc9a8)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906798 [select]:
github.com/segmentio/kafka-go.(*partitionWriter).awaitBatch(0xc0002060c0, 0xc0003485b0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1044 +0x76
github.com/segmentio/kafka-go.(*partitionWriter).newWriteBatch.func1()
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1036 +0x33
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000020f18)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4257 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0000bd890, 0x31d)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0000bd880)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000dafe78, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000dafe60)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc0018c0bd0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 924 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000ab9f50, 0x107b)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000ab9f40)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc001607398, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc001607380)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc400, 0xc0006fb3d0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 929 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0007af390, 0x631)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0007af380)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0016078d8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0016078c0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc600, 0xc000563180)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 955 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000e3a050, 0x625)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000e3a040)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0015ae8b8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0015ae8a0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc600, 0xc001acc0c0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 3754 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc002cdb190, 0x1062)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc002cdb180)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0006a2018, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0006a2000)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc400, 0xc0049d9bb0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4528395 [chan receive]:
github.com/segmentio/kafka-go.(*conn).run(0xc002574050, 0xc001f12440, 0xc00010a0c0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1246 +0xa5
created by github.com/segmentio/kafka-go.(*connGroup).connect
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/transport.go:1224 +0x9a5

goroutine 4678 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000b39c90, 0x317)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000b39c80)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc002d06438, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc002d06420)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc001266a60)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 3838 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0049c53d0, 0x1063)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0049c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc002cd13f8, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc002cd13e0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc400, 0xc004926620)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906801 [select]:
github.com/segmentio/kafka-go.(*partitionWriter).awaitBatch(0xc0005df7a0, 0xc000348700)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1044 +0x76
github.com/segmentio/kafka-go.(*partitionWriter).newWriteBatch.func1()
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:1036 +0x33
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc100, 0xc000020fa8)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4005 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc003d0ab90, 0x31b)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc003d0ab80)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc003591d58, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc003591d40)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc700, 0xc0019fca50)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69

goroutine 4906794 [IO wait]:
internal/poll.runtime_pollWait(0x7f7ac38d7c50, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc00022c218, 0x72, 0x0, 0x1, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00022c200, 0xc0039ec3a1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc00022c200, 0xc0039ec3a1, 0x1, 0x1, 0x4, 0xc0008a3708, 0x44e9df)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000208228, 0xc0039ec3a1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
net/http.(*connReader).backgroundRead(0xc0039ec390)
	/usr/local/go/src/net/http/server.go:672 +0x58
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:668 +0xd5

goroutine 1131661 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc0022a42d0, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0022a42c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0006a2318, 0x0)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:921 +0x85
github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0006a2300)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:981 +0x45
github.com/segmentio/kafka-go.(*Writer).spawn.func1(0xc0001dc500, 0xc0005fe300)
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:535 +0x57
created by github.com/segmentio/kafka-go.(*Writer).spawn
	/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.34/writer.go:533 +0x69




Environment

  • go version: go1.16
  • gin version (or commit ref): gin@v1.7.2
  • operating system: alpine3.13
@zouzehe
Copy link
Author

zouzehe commented Nov 4, 2022

the main code of the timeout middleware

package gin

import (
	"context"
	ginfk "github.com/gin-gonic/gin"
	"net/http"
	"time"
	"sync"
	"bytes"
	"fmt"
)

var (
	defaultOptions Options
)

func init() {
	defaultOptions = Options{
		CallBack:         nil,
		DefaultMsg:       `{"code": -1, "msg":"http: Handler timeout"}`,
		Timeout:          3 * time.Second,
		ErrorHttpCode:    http.StatusServiceUnavailable,
		CallBackResponse: nil,
	}
}

func Timeout(opts ...Option) ginfk.HandlerFunc {
	return func(c *ginfk.Context) {
		// sync.Pool
		buffer := GetBuff()

		tw := &Writer{body: buffer, ResponseWriter: c.Writer,
			h: make(http.Header)}
		tw.Options = defaultOptions

		// Loop through each option
		for _, opt := range opts {
			// Call the option giving the instantiated
			opt(tw)
		}

		// wrap the request context with a timeout
		ctx, cancel := context.WithTimeout(c.Request.Context(), tw.Timeout)
		defer cancel()

		c.Request = c.Request.WithContext(ctx)

		// Channel capacity must be greater than 0.
		// Otherwise, if the parent coroutine quit due to timeout,
		// the child coroutine may never be able to quit.
		finish := make(chan struct{}, 1)
		panicChan := make(chan interface{}, 1)
		go func() {
			defer func() {
				if p := recover(); p != nil {
					panicChan <- p
				}
			}()
			c.Next()
			finish <- struct{}{}
		}()

		var err error
		select {
		case p := <-panicChan:
			panic(p)

		case <-ctx.Done():
			tw.mu.Lock()
			defer tw.mu.Unlock()

			c.Writer = tw

			tw.timedOut = true
			tw.ResponseWriter.WriteHeader(tw.ErrorHttpCode)
			if tw.CallBackResponse != nil {
				body, ct := tw.CallBackResponse()
				head := tw.ResponseWriter.Header()
				head["Content-Type"] = ct
				_, err = tw.ResponseWriter.Write(body)
			} else {
				_, err = tw.ResponseWriter.Write([]byte(tw.DefaultMsg))
			}
			if err != nil {
				panic(err)
			}
			c.Abort()

			// execute callback func
			if tw.CallBack != nil {
				tw.CallBack(c.Request.Clone(context.Background()))
			}
			// If timeout happen, the buffer cannot be cleared actively,
			// but wait for the GC to recycle.
		case <-finish:
			tw.mu.Lock()
			defer tw.mu.Unlock()

			c.Writer = tw

			dst := tw.ResponseWriter.Header()
			for k, vv := range tw.Header() {
				dst[k] = vv
			}

			//if !tw.wroteHeader {
			//	tw.code = http.StatusOK
			//}

			tw.ResponseWriter.WriteHeader(tw.code)
			_, err = tw.ResponseWriter.Write(buffer.Bytes())
			if err != nil {
				panic(err)
			}
			PutBuff(buffer)
		}

	}
}




type Writer struct {
	ginfk.ResponseWriter
	// header
	h http.Header
	// body
	body    *bytes.Buffer
	Options // TimeoutOptions in options.go

	code        int
	mu          sync.Mutex
	timedOut    bool
	wroteHeader bool
}

func (tw *Writer) Write(b []byte) (int, error) {
	tw.mu.Lock()
	defer tw.mu.Unlock()
	if tw.timedOut {
		return 0, nil
	}

	return tw.body.Write(b)
}

func (tw *Writer) WriteHeader(code int) {
	checkWriteHeaderCode(code)
	tw.mu.Lock()
	defer tw.mu.Unlock()
	if tw.timedOut {
		return
	}
	tw.writeHeader(code)
}

func (tw *Writer) writeHeader(code int) {
	tw.wroteHeader = true
	tw.code = code
}

func (tw *Writer) WriteHeaderNow() {}

func (tw *Writer) Header() http.Header {
	return tw.h
}

func checkWriteHeaderCode(code int) {
	if code < 100 || code > 999 {
		panic(fmt.Sprintf("invalid WriteHeader code %v", code))
	}
}

@bbdshow
Copy link

bbdshow commented Nov 17, 2022

tw.mu can't be locked http.ResponseWriter.Header() map. it can only lock you custom Writer struct.
so the c.Data() and the tw.Header() timeout not mutex

@bbdshow
Copy link

bbdshow commented Nov 17, 2022

when timeout, the header should be custom, or reference https://github.com/gin-contrib/timeout

you can reference net/http/server.go line:3316 (go ver:1.17.7) implementation.

func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) {
	ctx := h.testContext
	if ctx == nil {
		var cancelCtx context.CancelFunc
		ctx, cancelCtx = context.WithTimeout(r.Context(), h.dt)
		defer cancelCtx()
	}
	r = r.WithContext(ctx)
	done := make(chan struct{})
	tw := &timeoutWriter{
		w:   w,
		h:   make(Header),
		req: r,
	}
	panicChan := make(chan interface{}, 1)
	go func() {
		defer func() {
			if p := recover(); p != nil {
				panicChan <- p
			}
		}()
		h.handler.ServeHTTP(tw, r)
		close(done)
	}()
	select {
	case p := <-panicChan:
		panic(p)
	case <-done:
		tw.mu.Lock()
		defer tw.mu.Unlock()
		dst := w.Header()
		for k, vv := range tw.h {
			dst[k] = vv
		}
		if !tw.wroteHeader {
			tw.code = StatusOK
		}
		w.WriteHeader(tw.code)
		w.Write(tw.wbuf.Bytes())
	case <-ctx.Done():
		tw.mu.Lock()
		defer tw.mu.Unlock()
		w.WriteHeader(StatusServiceUnavailable)
		io.WriteString(w, h.errorBody())
		tw.timedOut = true
	}
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants