I230517 21:00:18.014826 1 util/log/file_sync_buffer.go:238 ⋮ [config] file created at: 2023/05/17 21:00:18 I230517 21:00:18.014848 1 util/log/file_sync_buffer.go:238 ⋮ [config] running on machine: ‹rc-1684352534-06-n5cpu4-0004› I230517 21:00:18.014858 1 util/log/file_sync_buffer.go:238 ⋮ [config] binary: CockroachDB CCL v22.2.9 (x86_64-pc-linux-gnu, built 2023/05/08 12:58:16, go1.19.6) I230517 21:00:18.014867 1 util/log/file_sync_buffer.go:238 ⋮ [config] arguments: [‹./v22.2.9/cockroach› ‹start› ‹--insecure› ‹--log› ‹file-defaults: {dir: 'logs', exit-on-error: false}› ‹--listen-addr=:26257› ‹--http-addr=:26258› ‹--advertise-addr=10.142.0.107:26257› ‹--join=34.138.43.178:26257› ‹--store› ‹path=/mnt/data1/cockroach,attrs=store1:node4:node4store1› ‹--enterprise-encryption› ‹path=/mnt/data1/cockroach,key=/mnt/data1/cockroach/aes-128.key,old-key=plain› ‹--cache=25%› ‹--locality=cloud=gce,region=us-east1,zone=us-east1-b› ‹--max-sql-memory=25%›] I230517 21:00:18.014898 1 util/log/file_sync_buffer.go:238 ⋮ [config] log format (utf8=✓): crdb-v2 I230517 21:00:18.014903 1 util/log/file_sync_buffer.go:238 ⋮ [config] line format: [IWEF]yymmdd hh:mm:ss.uuuuuu goid [chan@]file:line redactionmark \[tags\] [counter] msg I230517 21:00:18.014400 1 util/log/flags.go:211 [-] 1 stderr capture started panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x58 pc=0x4127782] goroutine 498427 [running]: panic({0x4bb3320, 0x8fb3dd0}) GOROOT/src/runtime/panic.go:987 +0x3ba fp=0xc00b585df8 sp=0xc00b585d38 pc=0x48d77a runtime.panicmem(...) GOROOT/src/runtime/panic.go:260 runtime.sigpanic() GOROOT/src/runtime/signal_unix.go:835 +0x2f6 fp=0xc00b585e48 sp=0xc00b585df8 pc=0x4a4636 cloud.google.com/go/storage.(*Writer).open.func1() cloud.google.com/go/storage/external/com_google_cloud_go_storage/writer.go:162 +0x1a2 fp=0xc00b585fe0 sp=0xc00b585e48 pc=0x4127782 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b585fe8 sp=0xc00b585fe0 pc=0x4c2401 created by cloud.google.com/go/storage.(*Writer).open cloud.google.com/go/storage/external/com_google_cloud_go_storage/writer.go:152 +0x495 goroutine 1 [select, 9 minutes]: runtime.gopark(0xc0028b55e0?, 0x4?, 0x0?, 0x0?, 0xc0028b54c0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028b52f8 sp=0xc0028b52d8 pc=0x4907d6 runtime.selectgo(0xc0028b55e0, 0xc0028b54b8, 0xc0001a0000?, 0x0, 0xf?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028b5438 sp=0xc0028b52f8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/cli.waitForShutdown(0xc000016420, 0xc002a40360, 0xc0028a2a20, 0xc000809980, 0xc001fb2320) github.com/cockroachdb/cockroach/pkg/cli/start.go:729 +0x1b8 fp=0xc0028b5638 sp=0xc0028b5438 pc=0x4364b18 github.com/cockroachdb/cockroach/pkg/cli.runStart(0x9016280, {0xc000844c60?, 0xc000844ba0?, 0xc000c26ba0?}, 0x0) github.com/cockroachdb/cockroach/pkg/cli/start.go:677 +0x878 fp=0xc0028b59e0 sp=0xc0028b5638 pc=0x4363678 github.com/cockroachdb/cockroach/pkg/cli.runStartJoin(0xc001a4be40?, {0xc0005a9180?, 0xc000844d18?, 0x4?}) github.com/cockroachdb/cockroach/pkg/cli/start.go:341 +0x25 fp=0xc0028b5a18 sp=0xc0028b59e0 pc=0x4362dc5 github.com/cockroachdb/cockroach/pkg/cli/clierrorplus.MaybeDecorateError.func1(0xc000a6d200?, {0xc0005a9180?, 0xc0028b5ca8?, 0xc0028b5c80?}) github.com/cockroachdb/cockroach/pkg/cli/clierrorplus/decorate_error.go:71 +0x43 fp=0xc0028b5c90 sp=0xc0028b5a18 pc=0x4175263 github.com/cockroachdb/cockroach/pkg/cli/clierrorplus.MaybeShoutError.func1(0x9016280?, {0xc0005a9180?, 0x0?, 0xe?}) github.com/cockroachdb/cockroach/pkg/cli/clierrorplus/shout.go:24 +0x22 fp=0xc0028b5cd0 sp=0xc0028b5c90 pc=0x41764c2 github.com/spf13/cobra.(*Command).execute(0x9016280, {0xc0005a8fc0, 0xe, 0xe}) github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:856 +0x67c fp=0xc0028b5da8 sp=0xc0028b5cd0 pc=0x36b60fc github.com/spf13/cobra.(*Command).ExecuteC(0x9007c80) github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:974 +0x3bd fp=0xc0028b5e60 sp=0xc0028b5da8 pc=0x36b677d github.com/spf13/cobra.(*Command).Execute(...) github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:902 github.com/cockroachdb/cockroach/pkg/cli.Run(...) github.com/cockroachdb/cockroach/pkg/cli/cli.go:300 github.com/cockroachdb/cockroach/pkg/cli.doMain(0x9016280, {0xc001fb22ba, 0x5}) github.com/cockroachdb/cockroach/pkg/cli/cli.go:143 +0x1e9 fp=0xc0028b5ee0 sp=0xc0028b5e60 pc=0x4315f29 github.com/cockroachdb/cockroach/pkg/cli.Main() github.com/cockroachdb/cockroach/pkg/cli/cli.go:68 +0x157 fp=0xc0028b5f70 sp=0xc0028b5ee0 pc=0x4315bb7 main.main() main/pkg/cmd/cockroach/main.go:26 +0x17 fp=0xc0028b5f80 sp=0xc0028b5f70 pc=0x4497357 runtime.main() GOROOT/src/runtime/proc.go:250 +0x212 fp=0xc0028b5fe0 sp=0xc0028b5f80 pc=0x490412 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028b5fe8 sp=0xc0028b5fe0 pc=0x4c2401 goroutine 2 [force gc (idle), 9 minutes]: runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000098fb0 sp=0xc000098f90 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.forcegchelper() GOROOT/src/runtime/proc.go:302 +0xad fp=0xc000098fe0 sp=0xc000098fb0 pc=0x49066d runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000098fe8 sp=0xc000098fe0 pc=0x4c2401 created by runtime.init.6 GOROOT/src/runtime/proc.go:290 +0x25 goroutine 3 [GC sweep wait]: runtime.gopark(0x9bf4701?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000099790 sp=0xc000099770 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.bgsweep(0x0?) GOROOT/src/runtime/mgcsweep.go:297 +0xd7 fp=0xc0000997c8 sp=0xc000099790 pc=0x47ae37 runtime.gcenable.func1() GOROOT/src/runtime/mgc.go:178 +0x26 fp=0xc0000997e0 sp=0xc0000997c8 pc=0x46fa86 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000997e8 sp=0xc0000997e0 pc=0x4c2401 created by runtime.gcenable GOROOT/src/runtime/mgc.go:178 +0x6b goroutine 4 [GC scavenge wait]: runtime.gopark(0x4b7f1460e53?, 0x5b648d?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000099f70 sp=0xc000099f50 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.(*scavengerState).park(0x9bfc680) GOROOT/src/runtime/mgcscavenge.go:389 +0x53 fp=0xc000099fa0 sp=0xc000099f70 pc=0x478e93 runtime.bgscavenge(0x0?) GOROOT/src/runtime/mgcscavenge.go:622 +0x65 fp=0xc000099fc8 sp=0xc000099fa0 pc=0x479485 runtime.gcenable.func2() GOROOT/src/runtime/mgc.go:179 +0x26 fp=0xc000099fe0 sp=0xc000099fc8 pc=0x46fa26 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000099fe8 sp=0xc000099fe0 pc=0x4c2401 created by runtime.gcenable GOROOT/src/runtime/mgc.go:179 +0xaa goroutine 18 [finalizer wait]: runtime.gopark(0x0?, 0x5689fe0?, 0x90?, 0x7b?, 0x1000000010?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000098628 sp=0xc000098608 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.runfinq() GOROOT/src/runtime/mfinal.go:180 +0x10f fp=0xc0000987e0 sp=0xc000098628 pc=0x46eb8f runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000987e8 sp=0xc0000987e0 pc=0x4c2401 created by runtime.createfing GOROOT/src/runtime/mfinal.go:157 +0x45 goroutine 19 [chan receive]: runtime.gopark(0xcc0386?, 0xc000018000?, 0x38?, 0x8f?, 0xcc02f6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0000a8ec0 sp=0xc0000a8ea0 pc=0x4907d6 runtime.chanrecv(0xc0000c4060, 0xc0000a8fb8, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0000a8f50 sp=0xc0000a8ec0 pc=0x45a91b runtime.chanrecv2(0x9bff538?, 0xc0000a8fa0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0000a8f78 sp=0xc0000a8f50 pc=0x45a458 github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon() github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x5a fp=0xc0000a8fe0 sp=0xc0000a8f78 pc=0xcbe6fa runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000a8fe8 sp=0xc0000a8fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.init.5 github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x25 goroutine 20 [chan receive, 9 minutes]: runtime.gopark(0xc000094e68?, 0xc000094ec0?, 0x71?, 0xfa?, 0xc000094ec0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000094e90 sp=0xc000094e70 pc=0x4907d6 runtime.chanrecv(0xc00010e000, 0xc000094fb0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc000094f20 sp=0xc000094e90 pc=0x45a91b runtime.chanrecv2(0xc00010e000?, 0xc000094fc0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc000094f48 sp=0xc000094f20 pc=0x45a458 github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher() github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0xdc fp=0xc000094fe0 sp=0xc000094f48 pc=0xcbe93c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000094fe8 sp=0xc000094fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.init.5 github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x31 goroutine 34 [select, 9 minutes, locked to thread]: runtime.gopark(0xc00011e7a8?, 0x2?, 0x57?, 0xb?, 0xc00011e7a4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011e618 sp=0xc00011e5f8 pc=0x4907d6 runtime.selectgo(0xc00011e7a8, 0xc00011e7a0, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011e758 sp=0xc00011e618 pc=0x4a071c runtime.ensureSigM.func1() GOROOT/src/runtime/signal_unix.go:991 +0x1b0 fp=0xc00011e7e0 sp=0xc00011e758 pc=0x4a4b50 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011e7e8 sp=0xc00011e7e0 pc=0x4c2401 created by runtime.ensureSigM GOROOT/src/runtime/signal_unix.go:974 +0xbd goroutine 50 [syscall, 9 minutes]: runtime.notetsleepg(0x0?, 0x0?) GOROOT/src/runtime/lock_futex.go:236 +0x34 fp=0xc00011a7a0 sp=0xc00011a768 pc=0x4600b4 os/signal.signal_recv() GOROOT/src/runtime/sigqueue.go:152 +0x2f fp=0xc00011a7c0 sp=0xc00011a7a0 pc=0x4be76f os/signal.loop() GOROOT/src/os/signal/signal_unix.go:23 +0x19 fp=0xc00011a7e0 sp=0xc00011a7c0 pc=0xc9fe99 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011a7e8 sp=0xc00011a7e0 pc=0x4c2401 created by os/signal.Notify.func1.1 GOROOT/src/os/signal/signal.go:151 +0x2a goroutine 21 [GC worker (idle)]: runtime.gopark(0x4b90a9f0217?, 0x1?, 0xf7?, 0xdd?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000095750 sp=0xc000095730 pc=0x4907d6 runtime.gcBgMarkWorker() GOROOT/src/runtime/mgc.go:1235 +0xf1 fp=0xc0000957e0 sp=0xc000095750 pc=0x471bd1 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000957e8 sp=0xc0000957e0 pc=0x4c2401 created by runtime.gcBgMarkStartWorkers GOROOT/src/runtime/mgc.go:1159 +0x25 goroutine 5 [GC worker (idle)]: runtime.gopark(0x4b90aa290c9?, 0x1?, 0xeb?, 0x7e?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00009a750 sp=0xc00009a730 pc=0x4907d6 runtime.gcBgMarkWorker() GOROOT/src/runtime/mgc.go:1235 +0xf1 fp=0xc00009a7e0 sp=0xc00009a750 pc=0x471bd1 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00009a7e8 sp=0xc00009a7e0 pc=0x4c2401 created by runtime.gcBgMarkStartWorkers GOROOT/src/runtime/mgc.go:1159 +0x25 goroutine 22 [GC worker (idle)]: runtime.gopark(0x9c624c0?, 0x1?, 0x93?, 0xe8?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000095f50 sp=0xc000095f30 pc=0x4907d6 runtime.gcBgMarkWorker() GOROOT/src/runtime/mgc.go:1235 +0xf1 fp=0xc000095fe0 sp=0xc000095f50 pc=0x471bd1 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x4c2401 created by runtime.gcBgMarkStartWorkers GOROOT/src/runtime/mgc.go:1159 +0x25 goroutine 51 [GC worker (idle)]: runtime.gopark(0x4b90a8ee7cb?, 0x1?, 0xd5?, 0xc1?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011af50 sp=0xc00011af30 pc=0x4907d6 runtime.gcBgMarkWorker() GOROOT/src/runtime/mgc.go:1235 +0xf1 fp=0xc00011afe0 sp=0xc00011af50 pc=0x471bd1 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011afe8 sp=0xc00011afe0 pc=0x4c2401 created by runtime.gcBgMarkStartWorkers GOROOT/src/runtime/mgc.go:1159 +0x25 goroutine 4024 [runnable]: runtime.gopark(0xc0013ffa08?, 0x2?, 0xb0?, 0x0?, 0xc0013ff9ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0013ff870 sp=0xc0013ff850 pc=0x4907d6 runtime.selectgo(0xc0013ffa08, 0xc0013ff9e8, 0x16e9e6c?, 0x0, 0xc00a391f95?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0013ff9b0 sp=0xc0013ff870 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc004e36a50, {0xc0051814e0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc0013ffa38 sp=0xc0013ff9b0 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc004e36a50, {0xc0051814e0?, 0xc002b4bdb8?, 0xc0013ffa80?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc0013ffa78 sp=0xc0013ffa38 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0051b2e40, {0xc0051814e0?, 0xc0013ffaf8?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc0013ffac0 sp=0xc0013ffa78 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc0051b2e40}, {0xc0051814e0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc0013ffb08 sp=0xc0013ffac0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00512d7a0, {0xc0051814e0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc0013ffb48 sp=0xc0013ffb08 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc0051814d0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc0013ffbc8 sp=0xc0013ffb48 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x4908c0?, 0xc00512d7a0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc0013ffc60 sp=0xc0013ffbc8 pc=0xc19546 google.golang.org/grpc.recv(0xc0013ffd10?, {0x7f275f301038, 0xc0000af170}, 0x0?, {0x0?, 0x0?}, {0x512da60, 0xc01052b770}, 0xbf?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc0013ffcd8 sp=0xc0013ffc60 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc0051bc000, {0x512da60?, 0xc01052b770}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc0013ffe08 sp=0xc0013ffcd8 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x4618e7?, {0x512da60?, 0xc01052b770?}) :1 +0x34 fp=0xc0013ffe30 sp=0xc0013ffe08 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00605dd40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:263 +0x4c fp=0xc0013ffe60 sp=0xc0013ffe30 pc=0x1d4b66c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:91 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x628a958?, 0xc00605dd40}, 0xc000c62700, {0x624a948, 0xc0051b3a40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:289 +0xa6 fp=0xc0013ffee0 sp=0xc0013ffe60 pc=0x1d82d66 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x624a948?, 0xc0051b3a40?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:308 +0x45 fp=0xc0013fff30 sp=0xc0013ffee0 pc=0x1d82c65 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0013fffe0 sp=0xc0013fff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0013fffe8 sp=0xc0013fffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 8 [runnable]: runtime.gopark(0xc001578e50?, 0x3a214d2?, 0xc0?, 0xe2?, 0x9bf9280?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001578e20 sp=0xc001578e00 pc=0x4907d6 runtime.chanrecv(0xc000808060, 0xc001578f70, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc001578eb0 sp=0xc001578e20 pc=0x45a91b runtime.chanrecv1(0xc001578f88?, 0xc1116e636195171b?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc001578ed8 sp=0xc001578eb0 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1() github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x125 fp=0xc001578fe0 sp=0xc001578ed8 pc=0x22fae85 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001578fe8 sp=0xc001578fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0 github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x25 goroutine 23 [select]: runtime.gopark(0xc000121fa8?, 0x2?, 0xc0?, 0x24?, 0xc000121f74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000121dc8 sp=0xc000121da8 pc=0x4907d6 runtime.selectgo(0xc000121fa8, 0xc000121f70, 0x9bf9280?, 0x0, 0x78707d67da?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc000121f08 sp=0xc000121dc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc000121fe0 sp=0xc000121f08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000121fe8 sp=0xc000121fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 110 [select]: runtime.gopark(0xc00011b7a8?, 0x2?, 0x0?, 0x30?, 0xc00011b774?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011b5c8 sp=0xc00011b5a8 pc=0x4907d6 runtime.selectgo(0xc00011b7a8, 0xc00011b770, 0x9bf9280?, 0x0, 0x2d38e9d38c?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011b708 sp=0xc00011b5c8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc00011b7e0 sp=0xc00011b708 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011b7e8 sp=0xc00011b7e0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 52 [select]: runtime.gopark(0xc00011f788?, 0x3?, 0x58?, 0xe9?, 0xc00011f772?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011f5f8 sp=0xc00011f5d8 pc=0x4907d6 runtime.selectgo(0xc00011f788, 0xc00011f76c, 0xc000611e80?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011f738 sp=0xc00011f5f8 pc=0x4a071c go.opencensus.io/stats/view.(*worker).start(0xc000611e80) go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:276 +0xad fp=0xc00011f7c8 sp=0xc00011f738 pc=0x3c7646d go.opencensus.io/stats/view.init.0.func1() go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:34 +0x26 fp=0xc00011f7e0 sp=0xc00011f7c8 pc=0x3c756e6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011f7e8 sp=0xc00011f7e0 pc=0x4c2401 created by go.opencensus.io/stats/view.init.0 go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:34 +0x8d goroutine 53 [select, 9 minutes]: runtime.gopark(0xc00011efb0?, 0x2?, 0x0?, 0x0?, 0xc00011ef94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011ee18 sp=0xc00011edf8 pc=0x4907d6 runtime.selectgo(0xc00011efb0, 0xc00011ef90, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011ef58 sp=0xc00011ee18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup.func1() github.com/cockroachdb/cockroach/pkg/cli/cli.go:332 +0x97 fp=0xc00011efe0 sp=0xc00011ef58 pc=0x43175f7 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011efe8 sp=0xc00011efe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup github.com/cockroachdb/cockroach/pkg/cli/cli.go:330 +0x10d goroutine 54 [select, 9 minutes]: runtime.gopark(0xc00011ffb0?, 0x2?, 0x0?, 0x0?, 0xc00011ff2c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011fd88 sp=0xc00011fd68 pc=0x4907d6 runtime.selectgo(0xc00011ffb0, 0xc00011ff28, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011fec8 sp=0xc00011fd88 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup.func2() github.com/cockroachdb/cockroach/pkg/cli/cli.go:354 +0xa5 fp=0xc00011ffe0 sp=0xc00011fec8 pc=0x4317005 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011ffe8 sp=0xc00011ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup github.com/cockroachdb/cockroach/pkg/cli/cli.go:352 +0x1ed goroutine 55 [chan receive, 9 minutes]: runtime.gopark(0x4ebd5a0?, 0xc000ab4030?, 0xa0?, 0x7?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000120638 sp=0xc000120618 pc=0x4907d6 runtime.chanrecv(0xc000808f60, 0xc000120770, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0001206c8 sp=0xc000120638 pc=0x45a91b runtime.chanrecv2(0xc00266c000?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0001206f0 sp=0xc0001206c8 pc=0x45a458 github.com/getsentry/sentry-go.(*HTTPTransport).worker(0xc002b4e000) github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:387 +0x12f fp=0xc0001207c8 sp=0xc0001206f0 pc=0x8fb60f github.com/getsentry/sentry-go.(*HTTPTransport).Configure.func1.1() github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:255 +0x26 fp=0xc0001207e0 sp=0xc0001207c8 pc=0x8fac06 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001207e8 sp=0xc0001207e0 pc=0x4c2401 created by github.com/getsentry/sentry-go.(*HTTPTransport).Configure.func1 github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:255 +0x5a goroutine 56 [select, 9 minutes]: runtime.gopark(0xc0000a9f88?, 0x2?, 0xa0?, 0x1?, 0xc0000a9f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0000a9e08 sp=0xc0000a9de8 pc=0x4907d6 runtime.selectgo(0xc0000a9f88, 0xc0000a9f80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0000a9f48 sp=0xc0000a9e08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc002b56c30, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc0000a9fb8 sp=0xc0000a9f48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func6() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:203 +0x2e fp=0xc0000a9fe0 sp=0xc0000a9fb8 pc=0xcaec2e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000a9fe8 sp=0xc0000a9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:203 +0x790 goroutine 57 [select, 9 minutes]: runtime.gopark(0xc00011c788?, 0x2?, 0x0?, 0x0?, 0xc00011c784?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011c608 sp=0xc00011c5e8 pc=0x4907d6 runtime.selectgo(0xc00011c788, 0xc00011c780, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011c748 sp=0xc00011c608 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0007c60d0, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc00011c7b8 sp=0xc00011c748 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc00011c7e0 sp=0xc00011c7b8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011c7e8 sp=0xc00011c7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 58 [select, 9 minutes]: runtime.gopark(0xc00011cf88?, 0x2?, 0x0?, 0x0?, 0xc00011cf84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011ce08 sp=0xc00011cde8 pc=0x4907d6 runtime.selectgo(0xc00011cf88, 0xc00011cf80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011cf48 sp=0xc00011ce08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0007c64e0, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc00011cfb8 sp=0xc00011cf48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc00011cfe0 sp=0xc00011cfb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011cfe8 sp=0xc00011cfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 59 [select, 9 minutes]: runtime.gopark(0xc00011d788?, 0x2?, 0x0?, 0x0?, 0xc00011d784?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011d608 sp=0xc00011d5e8 pc=0x4907d6 runtime.selectgo(0xc00011d788, 0xc00011d780, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011d748 sp=0xc00011d608 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0007c75f0, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc00011d7b8 sp=0xc00011d748 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc00011d7e0 sp=0xc00011d7b8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011d7e8 sp=0xc00011d7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 60 [select, 9 minutes]: runtime.gopark(0xc003635f88?, 0x2?, 0xa0?, 0x1?, 0xc003635f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003635e08 sp=0xc003635de8 pc=0x4907d6 runtime.selectgo(0xc003635f88, 0xc003635f80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003635f48 sp=0xc003635e08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0007c7c70, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc003635fb8 sp=0xc003635f48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc003635fe0 sp=0xc003635fb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003635fe8 sp=0xc003635fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 61 [select, 9 minutes]: runtime.gopark(0xc00697cf88?, 0x2?, 0xa0?, 0x1?, 0xc00697cf84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00697ce08 sp=0xc00697cde8 pc=0x4907d6 runtime.selectgo(0xc00697cf88, 0xc00697cf80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00697cf48 sp=0xc00697ce08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00034cf70, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc00697cfb8 sp=0xc00697cf48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc00697cfe0 sp=0xc00697cfb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00697cfe8 sp=0xc00697cfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 62 [select, 9 minutes]: runtime.gopark(0xc0000a7f88?, 0x2?, 0xa0?, 0x1?, 0xc0000a7f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0000a7e08 sp=0xc0000a7de8 pc=0x4907d6 runtime.selectgo(0xc0000a7f88, 0xc0000a7f80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0000a7f48 sp=0xc0000a7e08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000018000, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc0000a7fb8 sp=0xc0000a7f48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc0000a7fe0 sp=0xc0000a7fb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000a7fe8 sp=0xc0000a7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 63 [select, 9 minutes]: runtime.gopark(0xc000096f88?, 0x2?, 0x0?, 0x0?, 0xc000096f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000096e08 sp=0xc000096de8 pc=0x4907d6 runtime.selectgo(0xc000096f88, 0xc000096f80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc000096f48 sp=0xc000096e08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0000180d0, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc000096fb8 sp=0xc000096f48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc000096fe0 sp=0xc000096fb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000096fe8 sp=0xc000096fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 64 [select, 9 minutes]: runtime.gopark(0xc000097788?, 0x2?, 0x0?, 0x0?, 0xc000097784?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000097608 sp=0xc0000975e8 pc=0x4907d6 runtime.selectgo(0xc000097788, 0xc000097780, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc000097748 sp=0xc000097608 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0000181a0, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc0000977b8 sp=0xc000097748 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc0000977e0 sp=0xc0000977b8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000977e8 sp=0xc0000977e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 65 [select, 9 minutes]: runtime.gopark(0xc001129f88?, 0x2?, 0xa0?, 0x1?, 0xc001129f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001129e08 sp=0xc001129de8 pc=0x4907d6 runtime.selectgo(0xc001129f88, 0xc001129f80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001129f48 sp=0xc001129e08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000018340, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc001129fb8 sp=0xc001129f48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc001129fe0 sp=0xc001129fb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001129fe8 sp=0xc001129fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 66 [select, 9 minutes]: runtime.gopark(0xc00009af88?, 0x2?, 0x0?, 0x0?, 0xc00009af84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00009ae08 sp=0xc00009ade8 pc=0x4907d6 runtime.selectgo(0xc00009af88, 0xc00009af80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00009af48 sp=0xc00009ae08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000018410, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc00009afb8 sp=0xc00009af48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc00009afe0 sp=0xc00009afb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00009afe8 sp=0xc00009afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 67 [select, 9 minutes]: runtime.gopark(0xc00009b788?, 0x2?, 0x0?, 0x0?, 0xc00009b784?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00009b608 sp=0xc00009b5e8 pc=0x4907d6 runtime.selectgo(0xc00009b788, 0xc00009b780, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00009b748 sp=0xc00009b608 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0000184e0, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc00009b7b8 sp=0xc00009b748 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc00009b7e0 sp=0xc00009b7b8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00009b7e8 sp=0xc00009b7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 68 [select, 8 minutes]: runtime.gopark(0xc001c70f88?, 0x2?, 0xa0?, 0x1?, 0xc001c70f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001c70e08 sp=0xc001c70de8 pc=0x4907d6 runtime.selectgo(0xc001c70f88, 0xc001c70f80, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001c70f48 sp=0xc001c70e08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000018680, {0x624a8a0, 0xc000676800}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f fp=0xc001c70fb8 sp=0xc001c70f48 pc=0xcab6bf github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig.func7() github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x2e fp=0xc001c70fe0 sp=0xc001c70fb8 pc=0xcae90e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001c70fe8 sp=0xc001c70fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a goroutine 12 [select]: runtime.gopark(0xc00011bfb0?, 0x2?, 0xae?, 0x3c?, 0xc00011bf94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00011be20 sp=0xc00011be00 pc=0x4907d6 runtime.selectgo(0xc00011bfb0, 0xc00011bf90, 0xc0028a2a20?, 0x0, 0xc001b82e00?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00011bf60 sp=0xc00011be20 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc00011bfe0 sp=0xc00011bf60 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00011bfe8 sp=0xc00011bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 13 [select]: runtime.gopark(0xc00187bfb0?, 0x2?, 0x80?, 0xff?, 0xc00187bf84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00187bdf0 sp=0xc00187bdd0 pc=0x4907d6 runtime.selectgo(0xc00187bfb0, 0xc00187bf80, 0x0?, 0x0, 0x7fe8609776?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00187bf30 sp=0xc00187bdf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc00187bfe0 sp=0xc00187bf30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00187bfe8 sp=0xc00187bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 14 [select]: runtime.gopark(0xc0028ce7b0?, 0x2?, 0xae?, 0x3c?, 0xc0028ce794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028ce620 sp=0xc0028ce600 pc=0x4907d6 runtime.selectgo(0xc0028ce7b0, 0xc0028ce790, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028ce760 sp=0xc0028ce620 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc0028ce7e0 sp=0xc0028ce760 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028ce7e8 sp=0xc0028ce7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 15 [select]: runtime.gopark(0xc002d91fb0?, 0x2?, 0x80?, 0xff?, 0xc002d91f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d91df0 sp=0xc002d91dd0 pc=0x4907d6 runtime.selectgo(0xc002d91fb0, 0xc002d91f80, 0x0?, 0x0, 0x7fe8605d51?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d91f30 sp=0xc002d91df0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc002d91fe0 sp=0xc002d91f30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d91fe8 sp=0xc002d91fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 16 [select]: runtime.gopark(0xc00c249fb0?, 0x2?, 0x0?, 0x0?, 0xc00c249f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c249e20 sp=0xc00c249e00 pc=0x4907d6 runtime.selectgo(0xc00c249fb0, 0xc00c249f90, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c249f60 sp=0xc00c249e20 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc00c249fe0 sp=0xc00c249f60 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c249fe8 sp=0xc00c249fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 82 [select]: runtime.gopark(0xc0028cffb0?, 0x2?, 0x80?, 0xff?, 0xc0028cff84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028cfdf0 sp=0xc0028cfdd0 pc=0x4907d6 runtime.selectgo(0xc0028cffb0, 0xc0028cff80, 0x0?, 0x0, 0x7fe8606e65?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028cff30 sp=0xc0028cfdf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc0028cffe0 sp=0xc0028cff30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028cffe8 sp=0xc0028cffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 83 [select]: runtime.gopark(0xc0028d07b0?, 0x2?, 0x1?, 0x0?, 0xc0028d0794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028d0620 sp=0xc0028d0600 pc=0x4907d6 runtime.selectgo(0xc0028d07b0, 0xc0028d0790, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028d0760 sp=0xc0028d0620 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc0028d07e0 sp=0xc0028d0760 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028d07e8 sp=0xc0028d07e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 84 [select]: runtime.gopark(0xc0028d0fb0?, 0x2?, 0x6?, 0x0?, 0xc0028d0f84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028d0df0 sp=0xc0028d0dd0 pc=0x4907d6 runtime.selectgo(0xc0028d0fb0, 0xc0028d0f80, 0x0?, 0x0, 0x7fe85d4f8c?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028d0f30 sp=0xc0028d0df0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc0028d0fe0 sp=0xc0028d0f30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028d0fe8 sp=0xc0028d0fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 85 [select]: runtime.gopark(0xc0028d17b0?, 0x2?, 0x0?, 0x6?, 0xc0028d1794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028d1620 sp=0xc0028d1600 pc=0x4907d6 runtime.selectgo(0xc0028d17b0, 0xc0028d1790, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028d1760 sp=0xc0028d1620 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc0028d17e0 sp=0xc0028d1760 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028d17e8 sp=0xc0028d17e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 86 [select]: runtime.gopark(0xc0011edfb0?, 0x2?, 0x6?, 0x0?, 0xc0011edf84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011eddf0 sp=0xc0011eddd0 pc=0x4907d6 runtime.selectgo(0xc0011edfb0, 0xc0011edf80, 0x0?, 0x0, 0x7fe86047d3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0011edf30 sp=0xc0011eddf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc0011edfe0 sp=0xc0011edf30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011edfe8 sp=0xc0011edfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 87 [select]: runtime.gopark(0xc0028ca7b0?, 0x2?, 0x0?, 0x30?, 0xc0028ca794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028ca620 sp=0xc0028ca600 pc=0x4907d6 runtime.selectgo(0xc0028ca7b0, 0xc0028ca790, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028ca760 sp=0xc0028ca620 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc0028ca7e0 sp=0xc0028ca760 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028ca7e8 sp=0xc0028ca7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 88 [select]: runtime.gopark(0xc0028cafb0?, 0x2?, 0x6?, 0x0?, 0xc0028caf84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028cadf0 sp=0xc0028cadd0 pc=0x4907d6 runtime.selectgo(0xc0028cafb0, 0xc0028caf80, 0x0?, 0x0, 0x7fe8608974?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028caf30 sp=0xc0028cadf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc0028cafe0 sp=0xc0028caf30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028cafe8 sp=0xc0028cafe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 89 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc0028cb6f8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028cb5f8 sp=0xc0028cb5d8 pc=0x4907d6 runtime.chanrecv(0xc00097a120, 0xc0028cb6d0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0028cb688 sp=0xc0028cb5f8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0028cb6b0 sp=0xc0028cb688 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc00094dbc0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc0028cb708 sp=0xc0028cb6b0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc0028cb7a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0028cb778 sp=0xc0028cb708 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc0028cb7c8 sp=0xc0028cb778 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc0028cb7e0 sp=0xc0028cb7c8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028cb7e8 sp=0xc0028cb7e0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 90 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc0028cbef8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028cbdf8 sp=0xc0028cbdd8 pc=0x4907d6 runtime.chanrecv(0xc00097a180, 0xc0028cbed0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0028cbe88 sp=0xc0028cbdf8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0028cbeb0 sp=0xc0028cbe88 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc00094d9b0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc0028cbf08 sp=0xc0028cbeb0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc0028cbfa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0028cbf78 sp=0xc0028cbf08 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc0028cbfc8 sp=0xc0028cbf78 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc0028cbfe0 sp=0xc0028cbfc8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028cbfe8 sp=0xc0028cbfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 91 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc0028cc6f8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028cc5f8 sp=0xc0028cc5d8 pc=0x4907d6 runtime.chanrecv(0xc00097a240, 0xc0028cc6d0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0028cc688 sp=0xc0028cc5f8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0028cc6b0 sp=0xc0028cc688 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc000942060}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc0028cc708 sp=0xc0028cc6b0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc0028cc7a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0028cc778 sp=0xc0028cc708 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc0028cc7c8 sp=0xc0028cc778 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc0028cc7e0 sp=0xc0028cc7c8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028cc7e8 sp=0xc0028cc7e0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 92 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc0028ccef8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028ccdf8 sp=0xc0028ccdd8 pc=0x4907d6 runtime.chanrecv(0xc00097a300, 0xc0028cced0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0028cce88 sp=0xc0028ccdf8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0028cceb0 sp=0xc0028cce88 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc000942540}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc0028ccf08 sp=0xc0028cceb0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc0028ccfa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0028ccf78 sp=0xc0028ccf08 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc0028ccfc8 sp=0xc0028ccf78 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc0028ccfe0 sp=0xc0028ccfc8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028ccfe8 sp=0xc0028ccfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 93 [select]: runtime.gopark(0xc0028cd7a8?, 0x2?, 0x0?, 0x30?, 0xc0028cd76c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0028cd5c8 sp=0xc0028cd5a8 pc=0x4907d6 runtime.selectgo(0xc0028cd7a8, 0xc0028cd768, 0x9bf9280?, 0x0, 0x7f61330bc8?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0028cd708 sp=0xc0028cd5c8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:365 +0xe5 fp=0xc0028cd7e0 sp=0xc0028cd708 pc=0x10594e5 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0028cd7e8 sp=0xc0028cd7e0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:360 +0x7a goroutine 459992 [sleep]: runtime.gopark(0x4b92f4290e6?, 0x7fb8c6?, 0x69?, 0xf6?, 0x811b0216cf?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d761df0 sp=0xc00d761dd0 pc=0x4907d6 time.Sleep(0x147d1ef23) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00d761e30 sp=0xc00d761df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc031e01c50, 0x7fb8c6, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00d761e88 sp=0xc00d761e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc031e01c50, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00d761ec8 sp=0xc00d761e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00a858c80?, 0xc, 0xc00bbcbf70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00d761fa8 sp=0xc00d761ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00d761fe0 sp=0xc00d761fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d761fe8 sp=0xc00d761fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 134 [select]: runtime.gopark(0xc0106d0f00?, 0x2?, 0x3?, 0x0?, 0xc0106d0ee4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0106d0d60 sp=0xc0106d0d40 pc=0x4907d6 runtime.selectgo(0xc0106d0f00, 0xc0106d0ee0, 0xc0011eb6c8?, 0x0, 0xc4487c?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0106d0ea0 sp=0xc0106d0d60 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1({0xc0005d8380?, 0xc002c074f0?}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/infostore.go:192 +0x85 fp=0xc0106d0f30 sp=0xc0106d0ea0 pc=0x199dc85 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0106d0fe0 sp=0xc0106d0f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0106d0fe8 sp=0xc0106d0fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 137 [select]: runtime.gopark(0xc0000967a8?, 0x2?, 0x0?, 0x30?, 0xc00009676c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0000965c8 sp=0xc0000965a8 pc=0x4907d6 runtime.selectgo(0xc0000967a8, 0xc000096768, 0xc002c07500?, 0x0, 0xc000469040?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc000096708 sp=0xc0000965c8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:365 +0xe5 fp=0xc0000967e0 sp=0xc000096708 pc=0x10594e5 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000967e8 sp=0xc0000967e0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:360 +0x7a goroutine 136 [select, 9 minutes]: runtime.gopark(0xc0011e7648?, 0x2?, 0x3?, 0x20?, 0xc0011e7644?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011e74d0 sp=0xc0011e74b0 pc=0x4907d6 runtime.selectgo(0xc0011e7648, 0xc0011e7640, 0xc000015d20?, 0x0, 0xc0011e7698?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0011e7610 sp=0xc0011e74d0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).runPeriodicSnapshotsLoop(0xc000c50000?, 0xc000c50000, 0xc003fe6840, 0xc00266da40, 0xc0011e76e0?) github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:295 +0xec fp=0xc0011e76a8 sp=0xc0011e7610 pc=0xc53e0c github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).PeriodicSnapshotsLoop(0xc45d72?, 0x88?, 0xc00a19a7e0?) github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:261 +0xb5 fp=0xc0011e7708 sp=0xc0011e76a8 pc=0xc53c95 github.com/cockroachdb/cockroach/pkg/server.newSQLServer.func1({0xc001846700, 0xc00a19a830}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:432 +0x45 fp=0xc0011e7730 sp=0xc0011e7708 pc=0x39df725 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0011e77e0 sp=0xc0011e7730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011e77e8 sp=0xc0011e77e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 135 [select]: runtime.gopark(0xc0000abef8?, 0x2?, 0x0?, 0x30?, 0xc0000abebc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0000abd38 sp=0xc0000abd18 pc=0x4907d6 runtime.selectgo(0xc0000abef8, 0xc0000abeb8, 0xc0011ebfb8?, 0x0, 0x6268d28?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0000abe78 sp=0xc0000abd38 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/ccl/utilccl.UpdateMetricOnLicenseChange.func2({0x624a948, 0xc000aad050}) github.com/cockroachdb/cockroach/pkg/ccl/utilccl/license_check.go:159 +0x13c fp=0xc0000abf30 sp=0xc0000abe78 pc=0x3a58ddc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0000abfe0 sp=0xc0000abf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000abfe8 sp=0xc0000abfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1241 [select]: runtime.gopark(0xc0079c5ca0?, 0x5?, 0x20?, 0x48?, 0xc0079c5a66?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079c5898 sp=0xc0079c5878 pc=0x4907d6 runtime.selectgo(0xc0079c5ca0, 0xc0079c5a5c, 0xc00327aa40?, 0x0, 0xc00327aa40?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0079c59d8 sp=0xc0079c5898 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc0049538c0, {0x624a8a0, 0xc00327aa40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:323 +0x95f fp=0xc0079c5d60 sp=0xc0079c59d8 pc=0x326315f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x624a948?, 0xc0035bef00?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x27a fp=0xc0079c5f30 sp=0xc0079c5d60 pc=0x326255a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0079c5fe0 sp=0xc0079c5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079c5fe8 sp=0xc0079c5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 133 [chan receive, 9 minutes]: runtime.gopark(0x1?, 0x10?, 0x1?, 0x0?, 0x1?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004233638 sp=0xc004233618 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0042336c8 sp=0xc004233638 pc=0x45a91b runtime.chanrecv1(0xc0005d80c8?, 0xc0042336e8?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc0042336f0 sp=0xc0042336c8 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func4({0xc0005d8000, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:597 +0x45 fp=0xc004233730 sp=0xc0042336f0 pc=0x1658f85 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0042337e0 sp=0xc004233730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0042337e8 sp=0xc0042337e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 344533 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x108?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008c4cf58 sp=0xc008c4cf38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f300c18?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc008c4cf90 sp=0xc008c4cf58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300b30, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc008c4cfb0 sp=0xc008c4cf90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc01885d080?, 0xc00d46f000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc008c4cfd8 sp=0xc008c4cfb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01885d080, {0xc00d46f000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc008c4d058 sp=0xc008c4cfd8 pc=0x535efa net.(*netFD).Read(0xc01885d080, {0xc00d46f000?, 0x7b5a68?, 0xc01885d080?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc008c4d0a0 sp=0xc008c4d058 pc=0x7a20c9 net.(*conn).Read(0xc004081a00, {0xc00d46f000?, 0x164290f?, 0xc004081a00?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc008c4d0e8 sp=0xc008c4d0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc010cd9aa0, {0xc00d46f000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc008c4d130 sp=0xc008c4d0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03bb72b70, {0xc00d46f000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc008c4d1d0 sp=0xc008c4d130 pc=0x370f896 bufio.(*Reader).fill(0xc0118e7090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc008c4d208 sp=0xc008c4d1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0118e7090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc008c4d228 sp=0xc008c4d208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0118e7668?, {0x62295c8, 0xc0118e7090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc008c4d260 sp=0xc008c4d228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0118e7000, 0xc000a147f0, 0xc008c4d442, 0xc008c4d444, {0x623b718, 0xc01983eab0}, 0xc0059fa000, 0xc0033837f4, 0xc008c4d440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc008c4d398 sp=0xc008c4d260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0118e7000, {0x624a948, 0xc003e24450}, 0xc000a147e0, 0xc0059fa000, 0xc003e24030, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc008c4d768 sp=0xc008c4d398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc003e24450}, {0x628a538, 0xc010cd9aa0}, {{{0xc016bae009, 0x4}}, 0x1, 0xc003e241b0, 0xc003e242a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc008c4d8a8 sp=0xc008c4d768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc003e24450}, {0x628a538?, 0xc010cd9aa0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc008c4de20 sp=0xc008c4d8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00d065ef0?}, {0x628a538, 0xc010cd9aa0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc008c4deb0 sp=0xc008c4de20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00d065ef0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc008c4df30 sp=0xc008c4deb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc008c4dfe0 sp=0xc008c4df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008c4dfe8 sp=0xc008c4dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1026 [select, 9 minutes]: runtime.gopark(0xc006a46b70?, 0x2?, 0x0?, 0x0?, 0xc006a46b6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a469f8 sp=0xc006a469d8 pc=0x4907d6 runtime.selectgo(0xc006a46b70, 0xc006a46b68, 0x5380c0?, 0x0, 0xc006a46b48?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006a46b38 sp=0xc006a469f8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc006807e60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc006a46ba0 sp=0xc006a46b38 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0079fa6e0, {0x512dba0?, 0xc0035c07b0}, 0xc004234510?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc006a46ca8 sp=0xc006a46ba0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc006a46cd8 sp=0xc006a46ca8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc006807b00, 0xc0042345b0, 0xc004234580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc006a46d30 sp=0xc006a46cd8 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc006807b00, {0x512dba0?, 0xc0035c07b0?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc006a46de0 sp=0xc006a46d30 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00437af00, {0x512dba0?, 0xc0035c07b0?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc006a46e28 sp=0xc006a46de0 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc003451600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:132 +0x4c fp=0xc006a46e58 sp=0xc006a46e28 pc=0x1d4b00c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x628a900, 0xc003451600}, 0xc000c62700, {0x624a948, 0xc0035c0750}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:409 +0x5b fp=0xc006a46ee0 sp=0xc006a46e58 pc=0x1d843fb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x624a948?, 0xc0035c0750?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:424 +0x45 fp=0xc006a46f30 sp=0xc006a46ee0 pc=0x1d84345 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006a46fe0 sp=0xc006a46f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a46fe8 sp=0xc006a46fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 314 [runnable]: runtime.gopark(0xc006a44d80?, 0xb?, 0x0?, 0x0?, 0x2e6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a44a80 sp=0xc006a44a60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x534825?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc006a44ab8 sp=0xc006a44a80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300c20, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc006a44ad8 sp=0xc006a44ab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc005d84e00?, 0xc006a08000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc006a44b00 sp=0xc006a44ad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc005d84e00, {0xc006a08000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc006a44b80 sp=0xc006a44b00 pc=0x535efa net.(*netFD).Read(0xc005d84e00, {0xc006a08000?, 0x50c7cc0?, 0x489101?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc006a44bc8 sp=0xc006a44b80 pc=0x7a20c9 net.(*conn).Read(0xc000549ca0, {0xc006a08000?, 0xc01026f400?, 0x5710000001?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc006a44c10 sp=0xc006a44bc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc006205ec0, {0xc000c63620, 0x9, 0x100000003?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc006a44c48 sp=0xc006a44c10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc006205ec0}, {0xc000c63620, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc006a44c90 sp=0xc006a44c48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc000c63620?, 0x9?, 0xc0068c6660?}, {0x62052c0?, 0xc006205ec0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc006a44ce0 sp=0xc006a44c90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c635e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc006a44d90 sp=0xc006a44ce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc005ba6780) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc006a44fc8 sp=0xc006a44d90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc006a44fe0 sp=0xc006a44fc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a44fe8 sp=0xc006a44fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 4115 [select]: runtime.gopark(0xc00a237f00?, 0x2?, 0x9c?, 0x0?, 0xc00a237ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ae05d60 sp=0xc00ae05d40 pc=0x4907d6 runtime.selectgo(0xc00ae05f00, 0xc00a237ed0, 0xc004981e30?, 0x0, 0xc00c2e1400?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ae05ea0 sp=0xc00ae05d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00749e5a0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc00ae05f30 sp=0xc00ae05ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004d8df20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc00ae05f88 sp=0xc00ae05f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc00ae05fe0 sp=0xc00ae05f88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ae05fe8 sp=0xc00ae05fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 156 [select]: runtime.gopark(0xc00a871ea0?, 0x2?, 0x58?, 0x1d?, 0xc00a871e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a871d18 sp=0xc00a871cf8 pc=0x4907d6 runtime.selectgo(0xc00a871ea0, 0xc00a871e88, 0xc000a88680?, 0x0, 0xc000000001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a871e58 sp=0xc00a871d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f00380, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00a871f30 sp=0xc00a871e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a871fe0 sp=0xc00a871f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a871fe8 sp=0xc00a871fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 920 [select]: runtime.gopark(0xc00cc73ed8?, 0x4?, 0x40?, 0x53?, 0xc00cc73e60?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00697acb8 sp=0xc00697ac98 pc=0x4907d6 runtime.selectgo(0xc00697aed8, 0xc00cc73e58, 0xc0034243f0?, 0x0, 0xc005976ed8?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00697adf8 sp=0xc00697acb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1({0x624a948, 0xc0034243f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:725 +0x1db fp=0xc00697af30 sp=0xc00697adf8 pc=0x39bcf5b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00697afe0 sp=0xc00697af30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00697afe8 sp=0xc00697afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 497004 [sleep]: runtime.gopark(0x4b8bf38b85e?, 0xf22d06?, 0xe5?, 0x9f?, 0x80a7992c35?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001438df0 sp=0xc001438dd0 pc=0x4907d6 time.Sleep(0x14472ded5) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc001438e30 sp=0xc001438df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc011b13e60, 0xf22d06, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc001438e88 sp=0xc001438e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc011b13e60, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc001438ec8 sp=0xc001438e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc011f6db80?, 0x8, 0xc00a6fdfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc001438fa8 sp=0xc001438ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc001438fe0 sp=0xc001438fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001438fe8 sp=0xc001438fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 487144 [sleep]: runtime.gopark(0x4b8b270aacc?, 0x8019a4?, 0x83?, 0x95?, 0x808d6a21d5?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a6fddf0 sp=0xc00a6fddd0 pc=0x4907d6 time.Sleep(0x1370bde6c) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a6fde30 sp=0xc00a6fddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00bf67980, 0x8019a4, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a6fde88 sp=0xc00a6fde30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00bf67980, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a6fdec8 sp=0xc00a6fde88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00202e000?, 0x31, 0xc00da2afb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a6fdfa8 sp=0xc00a6fdec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a6fdfe0 sp=0xc00a6fdfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a6fdfe8 sp=0xc00a6fdfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 139 [select]: runtime.gopark(0xc0011e7fa8?, 0x2?, 0x0?, 0x30?, 0xc0011e7f74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011e7dc8 sp=0xc0011e7da8 pc=0x4907d6 runtime.selectgo(0xc0011e7fa8, 0xc0011e7f70, 0x20c5f6ea1f72cfaa?, 0x0, 0x14186c7a96e3f74b?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0011e7f08 sp=0xc0011e7dc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc0011e7fe0 sp=0xc0011e7f08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011e7fe8 sp=0xc0011e7fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 49 [chan receive, 9 minutes]: runtime.gopark(0x1?, 0x10?, 0xa8?, 0xee?, 0x4ad945?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00422ee50 sp=0xc00422ee30 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc00422eee0 sp=0xc00422ee50 pc=0x45a91b runtime.chanrecv1(0xc45d72?, 0x31?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc00422ef08 sp=0xc00422eee0 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func1({0xc003656000, 0xc0042046c0}) github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:86 +0x37 fp=0xc00422ef30 sp=0xc00422ef08 pc=0x39a5757 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00422efe0 sp=0xc00422ef30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00422efe8 sp=0xc00422efe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 48 [chan receive, 9 minutes]: runtime.gopark(0x18?, 0x1?, 0x10?, 0x0?, 0xc00422f6a8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00422f648 sp=0xc00422f628 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc00422f6d8 sp=0xc00422f648 pc=0x45a91b runtime.chanrecv1(0xc00422f720?, 0xc45d72?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc00422f700 sp=0xc00422f6d8 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func3({0xc001ba8380, 0xc004204780}) github.com/cockroachdb/cockroach/pkg/server/start_listen.go:108 +0x45 fp=0xc00422f730 sp=0xc00422f700 pc=0x39e5fe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00422f7e0 sp=0xc00422f730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00422f7e8 sp=0xc00422f7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 41 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc00422fef8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00422fdf8 sp=0xc00422fdd8 pc=0x4907d6 runtime.chanrecv(0xc00799faa0, 0xc00422fed0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc00422fe88 sp=0xc00422fdf8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc00422feb0 sp=0xc00422fe88 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc0083e35f0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc00422ff08 sp=0xc00422feb0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x621ffa8?, 0xc0042047e0?}}, 0xc00422ffa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00422ff78 sp=0xc00422ff08 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0042047e0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc00422ffc8 sp=0xc00422ff78 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc00422ffe0 sp=0xc00422ffc8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00422ffe8 sp=0xc00422ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 42 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc0042306f8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0042305f8 sp=0xc0042305d8 pc=0x4907d6 runtime.chanrecv(0xc00799fb00, 0xc0042306d0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc004230688 sp=0xc0042305f8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0042306b0 sp=0xc004230688 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc0083e3650}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc004230708 sp=0xc0042306b0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc0042307a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc004230778 sp=0xc004230708 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc0042307c8 sp=0xc004230778 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc0042307e0 sp=0xc0042307c8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0042307e8 sp=0xc0042307e0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 43 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc004230ef8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004230df8 sp=0xc004230dd8 pc=0x4907d6 runtime.chanrecv(0xc00799fb60, 0xc004230ed0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc004230e88 sp=0xc004230df8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc004230eb0 sp=0xc004230e88 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc0083e36b0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc004230f08 sp=0xc004230eb0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc004230fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc004230f78 sp=0xc004230f08 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc004230fc8 sp=0xc004230f78 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc004230fe0 sp=0xc004230fc8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004230fe8 sp=0xc004230fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 44 [chan receive, 9 minutes]: runtime.gopark(0x4b2da40?, 0x4bb35c0?, 0xe0?, 0xfd?, 0xc0042316f8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0042315f8 sp=0xc0042315d8 pc=0x4907d6 runtime.chanrecv(0xc00799fbc0, 0xc0042316d0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc004231688 sp=0xc0042315f8 pc=0x45a91b runtime.chanrecv2(0x0?, 0x0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0042316b0 sp=0xc004231688 pc=0x45a458 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x624a948, 0xc0083e3710}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f fp=0xc004231708 sp=0xc0042316b0 pc=0x116641f runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000844c0?, 0x0?, 0x0?}}, 0xc0042317a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc004231778 sp=0xc004231708 pc=0xfd3223 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 fp=0xc0042317c8 sp=0xc004231778 pc=0x1166358 github.com/cockroachdb/pebble.(*tableCacheShard).init.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0x26 fp=0xc0042317e0 sp=0xc0042317c8 pc=0x11662c6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0042317e8 sp=0xc0042317e0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef goroutine 45 [select]: runtime.gopark(0xc004231fa8?, 0x2?, 0x0?, 0x30?, 0xc004231f74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004231dc8 sp=0xc004231da8 pc=0x4907d6 runtime.selectgo(0xc004231fa8, 0xc004231f70, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004231f08 sp=0xc004231dc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc004231fe0 sp=0xc004231f08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004231fe8 sp=0xc004231fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 154 [sync.Cond.Wait, 9 minutes]: runtime.gopark(0xc0011e85b8?, 0x7f27639a0e78?, 0x80?, 0x0?, 0x7f278cfaa108?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011e8540 sp=0xc0011e8520 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc000306fe0, 0x0) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0011e8588 sp=0xc0011e8540 pc=0x4be30c sync.(*Cond).Wait(0xc0011e85e8?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0011e85c0 sp=0xc0011e8588 pc=0x4cabac github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:198 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000306f00, {0x4bb35c0, 0x9c5fde0}) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:415 +0x647 fp=0xc0011e86f8 sp=0xc0011e85c0 pc=0x10ad167 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop-fm({0x624a948?, 0xc000afd230?}) :1 +0x39 fp=0xc0011e8720 sp=0xc0011e86f8 pc=0x10b0099 runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000843c0?, 0xc003459080?, 0x0?}}, 0xc0011e87c0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0011e8790 sp=0xc0011e8720 pc=0xfd3223 github.com/cockroachdb/pebble/record.NewLogWriter.func2() github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:329 +0x5c fp=0xc0011e87e0 sp=0xc0011e8790 pc=0x10aca3c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011e87e8 sp=0xc0011e87e0 pc=0x4c2401 created by github.com/cockroachdb/pebble/record.NewLogWriter github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:328 +0x3ca goroutine 141 [select]: runtime.gopark(0xc0011e8fa8?, 0x2?, 0x0?, 0x30?, 0xc0011e8f74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011e8dc8 sp=0xc0011e8da8 pc=0x4907d6 runtime.selectgo(0xc0011e8fa8, 0xc0011e8f70, 0x4309856ebd73a8f0?, 0x0, 0xcfad5856ea813558?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0011e8f08 sp=0xc0011e8dc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc0011e8fe0 sp=0xc0011e8f08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011e8fe8 sp=0xc0011e8fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 157 [select]: runtime.gopark(0xc00fd0fea0?, 0x2?, 0x58?, 0xfd?, 0xc00fd0fe8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d75dd18 sp=0xc00d75dcf8 pc=0x4907d6 runtime.selectgo(0xc00d75dea0, 0xc00fd0fe88, 0xc000a88680?, 0x0, 0xc000000002?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00d75de58 sp=0xc00d75dd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f00700, 0x8064c329b630f3}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00d75df30 sp=0xc00d75de58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00d75dfe0 sp=0xc00d75df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d75dfe8 sp=0xc00d75dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 117878 [select]: runtime.gopark(0xc002c8ef48?, 0x2?, 0x6?, 0x0?, 0xc002c8eef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002c8ed50 sp=0xc002c8ed30 pc=0x4907d6 runtime.selectgo(0xc002c8ef48, 0xc002c8eef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002c8ee90 sp=0xc002c8ed50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00f8fef00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc002c8efc8 sp=0xc002c8ee90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc002c8efe0 sp=0xc002c8efc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002c8efe8 sp=0xc002c8efe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 126 [select]: runtime.gopark(0xc005a51f00?, 0x2?, 0x0?, 0x0?, 0xc005a51edc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a51d68 sp=0xc005a51d48 pc=0x4907d6 runtime.selectgo(0xc005a51f00, 0xc005a51ed8, 0xc004233f00?, 0x0, 0x62ad3d0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a51ea8 sp=0xc005a51d68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startEventIntake.func3({0xc0057a4a80?, 0x0?}) github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:188 +0xab fp=0xc005a51f30 sp=0xc005a51ea8 pc=0x3246c0b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005a51fe0 sp=0xc005a51f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a51fe8 sp=0xc005a51fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 127 [select]: runtime.gopark(0xc00cf55ee8?, 0x3?, 0x72?, 0x0?, 0xc00cf55e8a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aa0cd8 sp=0xc004aa0cb8 pc=0x4907d6 runtime.selectgo(0xc004aa0ee8, 0xc00cf55e84, 0xc0059274d0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004aa0e18 sp=0xc004aa0cd8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startResolver.func1({0x624a948, 0xc0059274d0}) github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:219 +0x210 fp=0xc004aa0f30 sp=0xc004aa0e18 pc=0x3247150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc004aa0fe0 sp=0xc004aa0f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aa0fe8 sp=0xc004aa0fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 128 [select, 9 minutes]: runtime.gopark(0xc0079c5ed0?, 0x2?, 0x0?, 0x8?, 0xc0079c5e64?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005fedcb8 sp=0xc005fedc98 pc=0x4907d6 runtime.selectgo(0xc005feded0, 0xc0079c5e60, 0xc000800270?, 0x0, 0xe?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005feddf8 sp=0xc005fedcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func4({0xc0057a5180, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:191 +0x125 fp=0xc005fedf30 sp=0xc005feddf8 pc=0x33b6f85 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005fedfe0 sp=0xc005fedf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005fedfe8 sp=0xc005fedfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 129 [select, 9 minutes]: runtime.gopark(0xc0011ed6b0?, 0x2?, 0x0?, 0x0?, 0xc0011ed67c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011ed4d0 sp=0xc0011ed4b0 pc=0x4907d6 runtime.selectgo(0xc0011ed6b0, 0xc0011ed678, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0011ed610 sp=0xc0011ed4d0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x624a948, 0xc005927590}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd fp=0xc0011ed730 sp=0xc0011ed610 pc=0x33b766d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0011ed7e0 sp=0xc0011ed730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011ed7e8 sp=0xc0011ed7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 162 [select, 9 minutes]: runtime.gopark(0xc000097eb0?, 0x2?, 0x0?, 0x0?, 0xc000097e7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001124cd0 sp=0xc001124cb0 pc=0x4907d6 runtime.selectgo(0xc001124eb0, 0xc000097e78, 0xc000097f38?, 0x0, 0xc000018340?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001124e10 sp=0xc001124cd0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x624a948, 0xc0059275f0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd fp=0xc001124f30 sp=0xc001124e10 pc=0x33b766d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001124fe0 sp=0xc001124f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001124fe8 sp=0xc001124fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 163 [select, 9 minutes]: runtime.gopark(0xc0000946b0?, 0x2?, 0x8?, 0x0?, 0xc00009467c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001125cd0 sp=0xc001125cb0 pc=0x4907d6 runtime.selectgo(0xc001125eb0, 0xc000094678, 0x4618e7?, 0x0, 0x522f220?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001125e10 sp=0xc001125cd0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x624a948, 0xc005927650}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd fp=0xc001125f30 sp=0xc001125e10 pc=0x33b766d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001125fe0 sp=0xc001125f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001125fe8 sp=0xc001125fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 164 [select, 9 minutes]: runtime.gopark(0xc005986eb0?, 0x2?, 0x0?, 0x0?, 0xc005986e7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a5ecd0 sp=0xc005a5ecb0 pc=0x4907d6 runtime.selectgo(0xc005a5eeb0, 0xc005986e78, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a5ee10 sp=0xc005a5ecd0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x624a948, 0xc0059276b0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd fp=0xc005a5ef30 sp=0xc005a5ee10 pc=0x33b766d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005a5efe0 sp=0xc005a5ef30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a5efe8 sp=0xc005a5efe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 158 [select]: runtime.gopark(0xc00c4d1ea0?, 0x2?, 0x58?, 0x1d?, 0xc00c4d1e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c4d1d18 sp=0xc00c4d1cf8 pc=0x4907d6 runtime.selectgo(0xc00c4d1ea0, 0xc00c4d1e88, 0xc000a88680?, 0x0, 0xc000000003?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c4d1e58 sp=0xc00c4d1d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f00a80, 0xc0058a9da0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00c4d1f30 sp=0xc00c4d1e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c4d1fe0 sp=0xc00c4d1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c4d1fe8 sp=0xc00c4d1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 159 [select]: runtime.gopark(0xc00c70dea0?, 0x2?, 0x58?, 0xdd?, 0xc00c70de8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002e81d18 sp=0xc002e81cf8 pc=0x4907d6 runtime.selectgo(0xc002e81ea0, 0xc00c70de88, 0xc000a88680?, 0x0, 0xc000000001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002e81e58 sp=0xc002e81d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f00e00, 0xc0058a9ec0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc002e81f30 sp=0xc002e81e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002e81fe0 sp=0xc002e81f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002e81fe8 sp=0xc002e81fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 160 [select]: runtime.gopark(0xc00bf87ea0?, 0x2?, 0x58?, 0x7d?, 0xc00bf87e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bf87d18 sp=0xc00bf87cf8 pc=0x4907d6 runtime.selectgo(0xc00bf87ea0, 0xc00bf87e88, 0xc000a88680?, 0x0, 0xc000000001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bf87e58 sp=0xc00bf87d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f01500, 0x759f}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00bf87f30 sp=0xc00bf87e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00bf87fe0 sp=0xc00bf87f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bf87fe8 sp=0xc00bf87fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 161 [select]: runtime.gopark(0xc00ca63ea0?, 0x2?, 0x58?, 0x3d?, 0xc00ca63e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ca63d18 sp=0xc00ca63cf8 pc=0x4907d6 runtime.selectgo(0xc00ca63ea0, 0xc00ca63e88, 0xc000a88680?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ca63e58 sp=0xc00ca63d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f01880, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00ca63f30 sp=0xc00ca63e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ca63fe0 sp=0xc00ca63f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ca63fe8 sp=0xc00ca63fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 178 [select]: runtime.gopark(0xc00a8fdea0?, 0x2?, 0x58?, 0xdd?, 0xc00a8fde8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a8fdd18 sp=0xc00a8fdcf8 pc=0x4907d6 runtime.selectgo(0xc00a8fdea0, 0xc00a8fde88, 0xc000a88680?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a8fde58 sp=0xc00a8fdd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000f01c00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00a8fdf30 sp=0xc00a8fde58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a8fdfe0 sp=0xc00a8fdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a8fdfe8 sp=0xc00a8fdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 179 [select]: runtime.gopark(0xc00c05dea0?, 0x2?, 0x58?, 0xdd?, 0xc00c05de8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c05dd18 sp=0xc00c05dcf8 pc=0x4907d6 runtime.selectgo(0xc00c05dea0, 0xc00c05de88, 0xc000a88680?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c05de58 sp=0xc00c05dd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b4000, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00c05df30 sp=0xc00c05de58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c05dfe0 sp=0xc00c05df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c05dfe8 sp=0xc00c05dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 180 [select]: runtime.gopark(0xc00cd8dea0?, 0x2?, 0x58?, 0xdd?, 0xc00cd8de8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cd8dd18 sp=0xc00cd8dcf8 pc=0x4907d6 runtime.selectgo(0xc00cd8dea0, 0xc00cd8de88, 0xc000a88680?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00cd8de58 sp=0xc00cd8dd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b4380, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00cd8df30 sp=0xc00cd8de58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cd8dfe0 sp=0xc00cd8df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cd8dfe8 sp=0xc00cd8dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 181 [select]: runtime.gopark(0xc00c5d3ea0?, 0x2?, 0x58?, 0x3d?, 0xc00c5d3e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c5d3d18 sp=0xc00c5d3cf8 pc=0x4907d6 runtime.selectgo(0xc00c5d3ea0, 0xc00c5d3e88, 0xc000a88680?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c5d3e58 sp=0xc00c5d3d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b4700, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00c5d3f30 sp=0xc00c5d3e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c5d3fe0 sp=0xc00c5d3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c5d3fe8 sp=0xc00c5d3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 182 [select]: runtime.gopark(0xc00a9e7ea0?, 0x2?, 0x58?, 0x7d?, 0xc00a9e7e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b78bd18 sp=0xc00b78bcf8 pc=0x4907d6 runtime.selectgo(0xc00b78bea0, 0xc00a9e7e88, 0xc000a88680?, 0x0, 0xc000000001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b78be58 sp=0xc00b78bd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b4a80, 0xc0059986c0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00b78bf30 sp=0xc00b78be58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b78bfe0 sp=0xc00b78bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b78bfe8 sp=0xc00b78bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 183 [select]: runtime.gopark(0xc0086d1ea0?, 0x2?, 0x58?, 0x1d?, 0xc0086d1e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0086d1d18 sp=0xc0086d1cf8 pc=0x4907d6 runtime.selectgo(0xc0086d1ea0, 0xc0086d1e88, 0xc000a88680?, 0x0, 0xc000000001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0086d1e58 sp=0xc0086d1d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b4e00, 0xc005998780}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc0086d1f30 sp=0xc0086d1e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0086d1fe0 sp=0xc0086d1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0086d1fe8 sp=0xc0086d1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 184 [select]: runtime.gopark(0xc00abcfea0?, 0x2?, 0x58?, 0xfd?, 0xc00abcfe8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00abcfd18 sp=0xc00abcfcf8 pc=0x4907d6 runtime.selectgo(0xc00abcfea0, 0xc00abcfe88, 0xc000a88680?, 0x0, 0xc000000001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00abcfe58 sp=0xc00abcfd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b5180, 0xc0059987e0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00abcff30 sp=0xc00abcfe58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00abcffe0 sp=0xc00abcff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00abcffe8 sp=0xc00abcffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 185 [select]: runtime.gopark(0xc01115fea0?, 0x2?, 0x58?, 0xfd?, 0xc01115fe8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a41d18 sp=0xc006a41cf8 pc=0x4907d6 runtime.selectgo(0xc006a41ea0, 0xc01115fe88, 0xc000a88680?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006a41e58 sp=0xc006a41d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b5500, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc006a41f30 sp=0xc006a41e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006a41fe0 sp=0xc006a41f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a41fe8 sp=0xc006a41fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 186 [select]: runtime.gopark(0xc00a5f9ea0?, 0x2?, 0x58?, 0x9d?, 0xc00a5f9e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a5f9d18 sp=0xc00a5f9cf8 pc=0x4907d6 runtime.selectgo(0xc00a5f9ea0, 0xc00a5f9e88, 0xc000a88680?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a5f9e58 sp=0xc00a5f9d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b5880, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00a5f9f30 sp=0xc00a5f9e58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a5f9fe0 sp=0xc00a5f9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a5f9fe8 sp=0xc00a5f9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 187 [select]: runtime.gopark(0xc00b02bea0?, 0x2?, 0x58?, 0xbd?, 0xc00b02be8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b02bd18 sp=0xc00b02bcf8 pc=0x4907d6 runtime.selectgo(0xc00b02bea0, 0xc00b02be88, 0xc000a88680?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b02be58 sp=0xc00b02bd18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0036b5c00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8 fp=0xc00b02bf30 sp=0xc00b02be58 pc=0x33b73b8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b02bfe0 sp=0xc00b02bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b02bfe8 sp=0xc00b02bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 165 [select, 9 minutes]: runtime.gopark(0xc0059876e8?, 0x3?, 0xf6?, 0xda?, 0xc0059876a2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a5fd10 sp=0xc005a5fcf0 pc=0x4907d6 runtime.selectgo(0xc005a5fee8, 0xc00598769c, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a5fe50 sp=0xc005a5fd10 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.startPurgeOldSessions.func1({0xc005977500, 0x0}) github.com/cockroachdb/cockroach/pkg/server/purge_auth_session.go:69 +0x1e5 fp=0xc005a5ff30 sp=0xc005a5fe50 pc=0x39cac05 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005a5ffe0 sp=0xc005a5ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a5ffe8 sp=0xc005a5ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 166 [select, 9 minutes]: runtime.gopark(0xc005987fa8?, 0x2?, 0x1?, 0x30?, 0xc005987f5c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005987dd0 sp=0xc005987db0 pc=0x4907d6 runtime.selectgo(0xc005987fa8, 0xc005987f58, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005987f10 sp=0xc005987dd0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1() github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:264 +0x15a fp=0xc005987fe0 sp=0xc005987f10 pc=0xcd165a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005987fe8 sp=0xc005987fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:256 +0x13d goroutine 167 [chan receive, 9 minutes]: runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0059885e0 sp=0xc0059885c0 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc005988670 sp=0xc0059885e0 pc=0x45a91b runtime.chanrecv1(0x0?, 0xc0059886b8?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc005988698 sp=0xc005988670 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2({0xc005977c00, 0x0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x56 fp=0xc005988730 sp=0xc005988698 pc=0x1643396 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0059887e0 sp=0xc005988730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0059887e8 sp=0xc0059887e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 168 [chan receive, 9 minutes]: runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005988e00 sp=0xc005988de0 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc005988e90 sp=0xc005988e00 pc=0x45a91b runtime.chanrecv1(0xc005988f00?, 0xc4487c?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc005988eb8 sp=0xc005988e90 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/server.(*httpServer).start.func1({0x624a948, 0xc005991560}) github.com/cockroachdb/cockroach/pkg/server/server_http.go:221 +0x48 fp=0xc005988f30 sp=0xc005988eb8 pc=0x39d7f48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005988fe0 sp=0xc005988f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005988fe8 sp=0xc005988fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 169 [IO wait]: runtime.gopark(0x200000003?, 0xc005972ea0?, 0x0?, 0x60?, 0xc001123c00?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001123b90 sp=0xc001123b70 pc=0x4907d6 runtime.netpollblock(0xc001123be0?, 0x1123c58?, 0xc0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc001123bc8 sp=0xc001123b90 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f27645399f8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc001123be8 sp=0xc001123bc8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0058fe600?, 0x6?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001123c10 sp=0xc001123be8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc0058fe600) GOROOT/src/internal/poll/fd_unix.go:614 +0x234 fp=0xc001123ca8 sp=0xc001123c10 pc=0x539e14 net.(*netFD).accept(0xc0058fe600) GOROOT/src/net/fd_unix.go:172 +0x35 fp=0xc001123d60 sp=0xc001123ca8 pc=0x7a4235 net.(*TCPListener).accept(0xc0058f58d8) GOROOT/src/net/tcpsock_posix.go:142 +0x28 fp=0xc001123d90 sp=0xc001123d60 pc=0x7bfa48 net.(*TCPListener).Accept(0xc0058f58d8) GOROOT/src/net/tcpsock.go:288 +0x3d fp=0xc001123dc0 sp=0xc001123d90 pc=0x7bec1d net/http.(*onceCloseListener).Accept(0xc012940f00?) :1 +0x2a fp=0xc001123dd8 sp=0xc001123dc0 pc=0x8decea net/http.(*Server).Serve(0xc0058dcff0, {0x6246ed8, 0xc0058f58d8}) GOROOT/src/net/http/server.go:3070 +0x385 fp=0xc001123f08 sp=0xc001123dd8 pc=0x8b8905 github.com/cockroachdb/cockroach/pkg/server.(*httpServer).start.func4({0xc005a0c380, 0x0}) github.com/cockroachdb/cockroach/pkg/server/server_http.go:272 +0x28 fp=0xc001123f30 sp=0xc001123f08 pc=0x39d7b68 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001123fe0 sp=0xc001123f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001123fe8 sp=0xc001123fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 428126 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00e41d640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00be8b5d0 sp=0xc00be8b5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc006878a50, 0x160) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00be8b618 sp=0xc00be8b5d0 pc=0x4be30c sync.(*Cond).Wait(0xc00d3b1000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00be8b650 sp=0xc00be8b618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00d3b1668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc00be8b708 sp=0xc00be8b650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00404d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc00be8bcb8 sp=0xc00be8b708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00404d300, {0x624a8a0, 0xc006878a80}, 0x4145ac0300000000?, 0xc003be3dc0?, 0xc01847c100?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00be8bd70 sp=0xc00be8bcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc006878a80?}, {0xc0059fa000?}, 0x3?, 0xc0057bb4d0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00be8be20 sp=0xc00be8bd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00be8bfe0 sp=0xc00be8be20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00be8bfe8 sp=0xc00be8bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 194 [select, 9 minutes]: runtime.gopark(0xc0079a6d80?, 0x3?, 0x85?, 0x4f?, 0xc0079a6cfa?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079a6b58 sp=0xc0079a6b38 pc=0x4907d6 runtime.selectgo(0xc0079a6d80, 0xc0079a6cf4, 0x4cd0fe?, 0x0, 0x1c?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0079a6c98 sp=0xc0079a6b58 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.(*loopbackListener).Accept(0xc0036050e0) github.com/cockroachdb/cockroach/pkg/server/loopback.go:49 +0x9c fp=0xc0079a6dc0 sp=0xc0079a6c98 pc=0x39b655c google.golang.org/grpc.(*Server).Serve(0xc00296b6c0, {0x6233528?, 0xc0036050e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:780 +0x475 fp=0xc0079a6f08 sp=0xc0079a6dc0 pc=0xc1c4f5 github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func2({0xc003656380, 0x0}) github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:94 +0x2b fp=0xc0079a6f30 sp=0xc0079a6f08 pc=0x39a56eb github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0079a6fe0 sp=0xc0079a6f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079a6fe8 sp=0xc0079a6fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 195 [select, 9 minutes]: runtime.gopark(0xc0079a5f98?, 0x2?, 0x0?, 0x0?, 0xc0079a5f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079a5e00 sp=0xc0079a5de0 pc=0x4907d6 runtime.selectgo(0xc0079a5f98, 0xc0079a5f78, 0xc0084c17e0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0079a5f40 sp=0xc0079a5e00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00365e040) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc0079a5fc8 sp=0xc0079a5f40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc0079a5fe0 sp=0xc0079a5fc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079a5fe8 sp=0xc0079a5fe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 271 [select, 9 minutes]: runtime.gopark(0xc004235fb0?, 0x2?, 0xd0?, 0x6e?, 0xc004235f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004235e18 sp=0xc004235df8 pc=0x4907d6 runtime.selectgo(0xc004235fb0, 0xc004235f90, 0xc00296b6c0?, 0x0, 0xc002b7c9c0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004235f58 sp=0xc004235e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc004235fe0 sp=0xc004235f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004235fe8 sp=0xc004235fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 197 [chan receive, 9 minutes]: runtime.gopark(0x621ffa8?, 0xc0036b27c0?, 0x5?, 0x0?, 0xc0059beb00?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0059b6e00 sp=0xc0059b6de0 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0059b6e90 sp=0xc0059b6e00 pc=0x45a91b runtime.chanrecv1(0xc0059b6f00?, 0xc4487c?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc0059b6eb8 sp=0xc0059b6e90 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func5({0x624a948, 0xc0036057a0}) github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:127 +0x48 fp=0xc0059b6f30 sp=0xc0059b6eb8 pc=0x39a54e8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0059b6fe0 sp=0xc0059b6f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0059b6fe8 sp=0xc0059b6fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 198 [chan receive, 6 minutes]: runtime.gopark(0xc008de93e0?, 0xc0079a7d08?, 0xa?, 0xba?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079a7cd8 sp=0xc0079a7cb8 pc=0x4907d6 runtime.chanrecv(0xc0079fea20, 0xc0079a7da0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0079a7d68 sp=0xc0079a7cd8 pc=0x45a91b runtime.chanrecv2(0x498b20?, 0xc008de93e0?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc0079a7d90 sp=0xc0079a7d68 pc=0x45a458 github.com/cockroachdb/cmux.muxListener.Accept(...) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 github.com/cockroachdb/cmux.(*muxListener).Accept(0x62afba0?) :1 +0x45 fp=0xc0079a7dc0 sp=0xc0079a7d90 pc=0x1642445 google.golang.org/grpc.(*Server).Serve(0xc00296b6c0, {0x6248348?, 0xc001591818}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:780 +0x475 fp=0xc0079a7f08 sp=0xc0079a7dc0 pc=0xc1c4f5 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func5.1({0xc003657180, 0x0}) github.com/cockroachdb/cockroach/pkg/server/start_listen.go:134 +0x28 fp=0xc0079a7f30 sp=0xc0079a7f08 pc=0x39e5ea8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0079a7fe0 sp=0xc0079a7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079a7fe8 sp=0xc0079a7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 199 [IO wait]: runtime.gopark(0x200000003?, 0xc005aa2000?, 0x0?, 0x85?, 0xc005a5ac28?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a5abb8 sp=0xc005a5ab98 pc=0x4907d6 runtime.netpollblock(0xc005a5ac08?, 0x5a5ac80?, 0xc0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc005a5abf0 sp=0xc005a5abb8 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539ae8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc005a5ac10 sp=0xc005a5abf0 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc000a6bd80?, 0x6?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc005a5ac38 sp=0xc005a5ac10 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc000a6bd80) GOROOT/src/internal/poll/fd_unix.go:614 +0x234 fp=0xc005a5acd0 sp=0xc005a5ac38 pc=0x539e14 net.(*netFD).accept(0xc000a6bd80) GOROOT/src/net/fd_unix.go:172 +0x35 fp=0xc005a5ad88 sp=0xc005a5acd0 pc=0x7a4235 net.(*TCPListener).accept(0xc0015917e8) GOROOT/src/net/tcpsock_posix.go:142 +0x28 fp=0xc005a5adb8 sp=0xc005a5ad88 pc=0x7bfa48 net.(*TCPListener).Accept(0xc0015917e8) GOROOT/src/net/tcpsock.go:288 +0x3d fp=0xc005a5ade8 sp=0xc005a5adb8 pc=0x7bec1d github.com/cockroachdb/cmux.(*cMux).Serve(0xc000a0fdc0) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9 fp=0xc005a5ae78 sp=0xc005a5ade8 pc=0x16414c9 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func5.2.1() github.com/cockroachdb/cockroach/pkg/server/start_listen.go:139 +0x22 fp=0xc005a5ae98 sp=0xc005a5ae78 pc=0x39e5e62 sync.(*Once).doSlow(0xc005985ee8?, 0xc005985f20?) GOROOT/src/sync/once.go:74 +0xc2 fp=0xc005a5aef8 sp=0xc005a5ae98 pc=0x4ccc42 sync.(*Once).Do(...) GOROOT/src/sync/once.go:65 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func5.2({0xc003657500, 0x0}) github.com/cockroachdb/cockroach/pkg/server/start_listen.go:138 +0x55 fp=0xc005a5af30 sp=0xc005a5aef8 pc=0x39e5e15 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005a5afe0 sp=0xc005a5af30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a5afe8 sp=0xc005a5afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3982 [select, 9 minutes]: runtime.gopark(0xc002f4b578?, 0x2?, 0x0?, 0x38?, 0xc002f4b53c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002f4b3a8 sp=0xc002f4b388 pc=0x4907d6 runtime.selectgo(0xc002f4b578, 0xc002f4b538, 0xc001660f80?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002f4b4e8 sp=0xc002f4b3a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000c62700, {0x628a958, 0xc003450d30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:313 +0x208 fp=0xc002f4b5b0 sp=0xc002f4b4e8 pc=0x1d82b08 github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x525f4e0?, 0xc000c62700}, {0x627d570?, 0xc0030d3da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:244 +0x9f fp=0xc002f4b5e8 sp=0xc002f4b5b0 pc=0x1d4b57f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc003d528f0?}, 0xc001664648, 0x5683c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc002f4b920 sp=0xc002f4b5e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc003d528f0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc002f4b960 sp=0xc002f4b920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc003d528f0}, 0x1054f90?, 0xc001660f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc002f4b990 sp=0xc002f4b960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc003d528f0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc002f4b9d0 sp=0xc002f4b990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc002f4ba00 sp=0xc002f4b9d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc001663d70}, {0xc001832b08?, 0x46159f?}, 0xc001832ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc002f4ba78 sp=0xc002f4ba00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc003d528f0?}, 0xc001664648, 0xc001660f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc002f4bb00 sp=0xc002f4ba78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc003d528f0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc002f4bb40 sp=0xc002f4bb00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc003d528f0}, 0xc001664648, 0x5683c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc002f4bb90 sp=0xc002f4bb40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc004caa680}, 0xc0050845a0, 0xc000ace270, 0x8ff1fe0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc002f4be48 sp=0xc002f4bb90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc004caa680}, 0xc0050845a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc002f4bf68 sp=0xc002f4be48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc002f4bfe0 sp=0xc002f4bf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002f4bfe8 sp=0xc002f4bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 201 [select]: runtime.gopark(0xc00608fee8?, 0x3?, 0xb8?, 0xa5?, 0xc00608fe72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00608fcc8 sp=0xc00608fca8 pc=0x4907d6 runtime.selectgo(0xc00608fee8, 0xc00608fe6c, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00608fe08 sp=0xc00608fcc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1({0x624a948, 0xc003605b60}) github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:143 +0x42f fp=0xc00608ff30 sp=0xc00608fe08 pc=0x390a98f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00608ffe0 sp=0xc00608ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00608ffe8 sp=0xc00608ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 202 [select]: runtime.gopark(0xc005a5be98?, 0x3?, 0x0?, 0x30?, 0xc005a5be52?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a5bcb0 sp=0xc005a5bc90 pc=0x4907d6 runtime.selectgo(0xc005a5be98, 0xc005a5be4c, 0xc007572b40?, 0x0, 0x3f0a36e2eb1c432d?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a5bdf0 sp=0xc005a5bcb0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1.1({0x624a948, 0xc003605c50}) github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:108 +0x2ec fp=0xc005a5bf30 sp=0xc005a5bdf0 pc=0x390b10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005a5bfe0 sp=0xc005a5bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a5bfe8 sp=0xc005a5bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 48622 [select]: runtime.gopark(0xc0088e1db8?, 0x3?, 0x4b?, 0x0?, 0xc0088e1d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0088e1b90 sp=0xc0088e1b70 pc=0x4907d6 runtime.selectgo(0xc0088e1db8, 0xc0088e1d2c, 0xc013d54120?, 0x0, 0xc003971221?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0088e1cd0 sp=0xc0088e1b90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc013d54120}, 0xc015b27770, {0xc007946a68, 0x12}, 0xc003f2a300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc0088e1e68 sp=0xc0088e1cd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc013d54120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc0088e1f30 sp=0xc0088e1e68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0088e1fe0 sp=0xc0088e1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0088e1fe8 sp=0xc0088e1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 204 [select]: runtime.gopark(0xc005a5db78?, 0x4?, 0xf8?, 0xd9?, 0xc005a5db38?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a5d9b0 sp=0xc005a5d990 pc=0x4907d6 runtime.selectgo(0xc005a5db78, 0xc005a5db30, 0x568de40?, 0x0, 0x487350?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a5daf0 sp=0xc005a5d9b0 pc=0x4a071c net.(*pipe).read(0xc000932b00, {0xc005d8a000, 0x8000, 0xc005a5dc07?}) GOROOT/src/net/pipe.go:159 +0x157 fp=0xc005a5dbc8 sp=0xc005a5daf0 pc=0x7b97b7 net.(*pipe).Read(0xc005aa2d00?, {0xc005d8a000?, 0xc005a5dc50?, 0x459e71?}) GOROOT/src/net/pipe.go:142 +0x25 fp=0xc005a5dc10 sp=0xc005a5dbc8 pc=0x7b9505 bufio.(*Reader).Read(0xc0079ff800, {0xc0079dc200, 0x9, 0x18?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc005a5dc48 sp=0xc005a5dc10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc0079ff800}, {0xc0079dc200, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc005a5dc90 sp=0xc005a5dc48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc0079dc200?, 0x9?, 0xc005a5dd37?}, {0x62052c0?, 0xc0079ff800?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc005a5dce0 sp=0xc005a5dc90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0079dc1c0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc005a5dd90 sp=0xc005a5dce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc005a8c3c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc005a5dfc8 sp=0xc005a5dd90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc005a5dfe0 sp=0xc005a5dfc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a5dfe8 sp=0xc005a5dfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 205 [select]: runtime.gopark(0xc00454ff00?, 0x2?, 0x88?, 0xfd?, 0xc00454fed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00454fd60 sp=0xc00454fd40 pc=0x4907d6 runtime.selectgo(0xc00454ff00, 0xc00454fed0, 0x7b9965?, 0x0, 0xc0080c2000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00454fea0 sp=0xc00454fd60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0008e6960, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc00454ff30 sp=0xc00454fea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0079ff920) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc00454ff88 sp=0xc00454ff30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc00454ffe0 sp=0xc00454ff88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00454ffe8 sp=0xc00454ffe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 206 [select]: runtime.gopark(0xc004267f00?, 0x2?, 0x88?, 0x7d?, 0xc004267ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004267d60 sp=0xc004267d40 pc=0x4907d6 runtime.selectgo(0xc004267f00, 0xc004267ed0, 0x7b9965?, 0x0, 0xc007fea000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004267ea0 sp=0xc004267d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0008e6820, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc004267f30 sp=0xc004267ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0079ffa40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc004267f88 sp=0xc004267f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc004267fe0 sp=0xc004267f88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004267fe8 sp=0xc004267fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 207 [select]: runtime.gopark(0xc005feaf70?, 0x4?, 0x6?, 0x0?, 0xc005feaed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005fead18 sp=0xc005feacf8 pc=0x4907d6 runtime.selectgo(0xc005feaf70, 0xc005feaec8, 0x6213da0?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005feae58 sp=0xc005fead18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc005aa29c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc005feafc8 sp=0xc005feae58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc005feafe0 sp=0xc005feafc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005feafe8 sp=0xc005feafe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 208 [select]: runtime.gopark(0xc005a5cbe0?, 0x4?, 0x60?, 0xca?, 0xc005a5cba0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a5ca18 sp=0xc005a5c9f8 pc=0x4907d6 runtime.selectgo(0xc005a5cbe0, 0xc005a5cb98, 0x0?, 0x0, 0x487350?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a5cb58 sp=0xc005a5ca18 pc=0x4a071c net.(*pipe).read(0xc000932c00, {0xc005d10000, 0x8000, 0x7?}) GOROOT/src/net/pipe.go:159 +0x157 fp=0xc005a5cc30 sp=0xc005a5cb58 pc=0x7b97b7 net.(*pipe).Read(0x800010601?, {0xc005d10000?, 0xc005aa31e0?, 0x800010601?}) GOROOT/src/net/pipe.go:142 +0x25 fp=0xc005a5cc78 sp=0xc005a5cc30 pc=0x7b9505 bufio.(*Reader).Read(0xc0079ff620, {0xc0079dc120, 0x9, 0x18?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc005a5ccb0 sp=0xc005a5cc78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc0079ff620}, {0xc0079dc120, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc005a5ccf8 sp=0xc005a5ccb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc0079dc120?, 0x9?, 0xc00413eaf8?}, {0x62052c0?, 0xc0079ff620?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc005a5cd48 sp=0xc005a5ccf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0079dc0e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc005a5cdf8 sp=0xc005a5cd48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc005aa29c0, 0xc005985fd0?, 0x1054c46?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc005a5cf18 sp=0xc005a5cdf8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc005aa29c0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc005a5cf80 sp=0xc005a5cf18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc005a5cfe0 sp=0xc005a5cf80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a5cfe8 sp=0xc005a5cfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 875 [chan receive, 9 minutes]: runtime.gopark(0x4570ac?, 0x6294e00?, 0x94?, 0x92?, 0xc000084440?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011e9e48 sp=0xc0011e9e28 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0011e9ed8 sp=0xc0011e9e48 pc=0x45a91b runtime.chanrecv1(0xc0011e9f20?, 0xc45d72?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc0011e9f00 sp=0xc0011e9ed8 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1({0xc003657880, 0xc0001a0000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:203 +0x3b fp=0xc0011e9f30 sp=0xc0011e9f00 pc=0x1e269db github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0011e9fe0 sp=0xc0011e9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011e9fe8 sp=0xc0011e9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1065 [select]: runtime.gopark(0xc007e2bef8?, 0x2?, 0x4?, 0x0?, 0xc007e2bec4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a8a5d20 sp=0xc00a8a5d00 pc=0x4907d6 runtime.selectgo(0xc00a8a5ef8, 0xc007e2bec0, 0xc003d67a10?, 0x0, 0x4570ac?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a8a5e60 sp=0xc00a8a5d20 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func2({0x624a948, 0xc003d67a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:958 +0x105 fp=0xc00a8a5f30 sp=0xc00a8a5e60 pc=0x39bee05 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a8a5fe0 sp=0xc00a8a5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a8a5fe8 sp=0xc00a8a5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 912 [select]: runtime.gopark(0xc00ba49db8?, 0x3?, 0x63?, 0x0?, 0xc00ba49d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ba49b90 sp=0xc00ba49b70 pc=0x4907d6 runtime.selectgo(0xc00ba49db8, 0xc00ba49d2c, 0xc0034ad8c0?, 0x0, 0xc002c27040?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ba49cd0 sp=0xc00ba49b90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc0034ad8c0}, 0xc005845ea0, {0xc005e768e8, 0x12}, 0xc0026c04e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc00ba49e68 sp=0xc00ba49cd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc0034ad8c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc00ba49f30 sp=0xc00ba49e68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ba49fe0 sp=0xc00ba49f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ba49fe8 sp=0xc00ba49fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 272 [select, 9 minutes]: runtime.gopark(0xc0042327b0?, 0x2?, 0x32?, 0x11?, 0xc00423279c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004232628 sp=0xc004232608 pc=0x4907d6 runtime.selectgo(0xc0042327b0, 0xc004232798, 0xc002b7c9c0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004232768 sp=0xc004232628 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0042327e0 sp=0xc004232768 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0042327e8 sp=0xc0042327e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 278 [chan receive, 9 minutes]: runtime.gopark(0x0?, 0x0?, 0x31?, 0x0?, 0x40000?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0059b8638 sp=0xc0059b8618 pc=0x4907d6 runtime.chanrecv(0xc00266da40, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0059b86c8 sp=0xc0059b8638 pc=0x45a91b runtime.chanrecv1(0xc00601a448?, 0xc0059b86e8?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc0059b86f0 sp=0xc0059b86c8 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3({0xc00601a380, 0x0}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/server.go:380 +0x49 fp=0xc0059b8730 sp=0xc0059b86f0 pc=0x19a3789 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0059b87e0 sp=0xc0059b8730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0059b87e8 sp=0xc0059b87e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 279 [select, 9 minutes]: runtime.gopark(0xc006090ed8?, 0x2?, 0x11?, 0x0?, 0xc006090e8c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006090d08 sp=0xc006090ce8 pc=0x4907d6 runtime.selectgo(0xc006090ed8, 0xc006090e88, 0x53ce5a3?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006090e48 sp=0xc006090d08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1({0x624a948?, 0xc006794930?}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1261 +0x259 fp=0xc006090f30 sp=0xc006090e48 pc=0x199a879 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006090fe0 sp=0xc006090f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006090fe8 sp=0xc006090fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 280 [select]: runtime.gopark(0xc002d8fec0?, 0x5?, 0xa8?, 0xfd?, 0xc002d8fe76?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d8fcd0 sp=0xc002d8fcb0 pc=0x4907d6 runtime.selectgo(0xc002d8fec0, 0xc002d8fe6c, 0xc006794990?, 0x0, 0x621ffa8?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d8fe10 sp=0xc002d8fcd0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1({0x624a948, 0xc006794990}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1293 +0x276 fp=0xc002d8ff30 sp=0xc002d8fe10 pc=0x199b236 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002d8ffe0 sp=0xc002d8ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d8ffe8 sp=0xc002d8ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 281 [select]: runtime.gopark(0xc003dcfe98?, 0x5?, 0x0?, 0x0?, 0xc003dcfdb6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003dcfc18 sp=0xc003dcfbf8 pc=0x4907d6 runtime.selectgo(0xc003dcfe98, 0xc003dcfdac, 0xc00666d900?, 0x0, 0xc0058a9da0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003dcfd58 sp=0xc003dcfc18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc005b2af20, {0x624a948, 0xc0067c4480?}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:475 +0x346 fp=0xc003dcff08 sp=0xc003dcfd58 pc=0x1ca1f66 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run-fm({0x624a948?, 0xc0067c4480?}) :1 +0x39 fp=0xc003dcff30 sp=0xc003dcff08 pc=0x1ca3079 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003dcffe0 sp=0xc003dcff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003dcffe8 sp=0xc003dcffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 282 [select]: runtime.gopark(0xc00a34fe98?, 0x5?, 0x0?, 0x54?, 0xc00a34fdb6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a34fc18 sp=0xc00a34fbf8 pc=0x4907d6 runtime.selectgo(0xc00a34fe98, 0xc00a34fdac, 0xc00666d940?, 0x0, 0x5?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a34fd58 sp=0xc00a34fc18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc005b2b080, {0x624a948, 0xc0067c4510?}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:475 +0x346 fp=0xc00a34ff08 sp=0xc00a34fd58 pc=0x1ca1f66 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run-fm({0x624a948?, 0xc0067c4510?}) :1 +0x39 fp=0xc00a34ff30 sp=0xc00a34ff08 pc=0x1ca3079 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a34ffe0 sp=0xc00a34ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a34ffe8 sp=0xc00a34ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 283 [select]: runtime.gopark(0xc007e59e98?, 0x5?, 0x9f?, 0x15?, 0xc007e59db6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b3c0c18 sp=0xc00b3c0bf8 pc=0x4907d6 runtime.selectgo(0xc00b3c0e98, 0xc007e59dac, 0xc00666d400?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b3c0d58 sp=0xc00b3c0c18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc005b2b1e0, {0x624a948, 0xc0067c45a0?}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:475 +0x346 fp=0xc00b3c0f08 sp=0xc00b3c0d58 pc=0x1ca1f66 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run-fm({0x624a948?, 0xc0067c45a0?}) :1 +0x39 fp=0xc00b3c0f30 sp=0xc00b3c0f08 pc=0x1ca3079 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b3c0fe0 sp=0xc00b3c0f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b3c0fe8 sp=0xc00b3c0fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498463 [select]: runtime.gopark(0xc00c70c2d8?, 0x2?, 0x30?, 0xaf?, 0xc00c70c2d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c70c160 sp=0xc00c70c140 pc=0x4907d6 runtime.selectgo(0xc00c70c2d8, 0xc00c70c2d0, 0x10?, 0x0, 0x24001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c70c2a0 sp=0xc00c70c160 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc013150ea0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00c70c308 sp=0xc00c70c2a0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018acc60, {0x51b9a20?, 0xc00d502880}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00c70c410 sp=0xc00c70c308 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x20183?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00c70c440 sp=0xc00c70c410 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0131506c0, 0xc00c70c518, 0xc00c70c4e8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00c70c498 sp=0xc00c70c440 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0131506c0, {0x51b9a20?, 0xc00d502880?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00c70c548 sp=0xc00c70c498 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03064aba0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0120d2690}, {0x51b9a20, 0xc00d502880}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00c70c5b0 sp=0xc00c70c548 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03064aba0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0120d2690}, {0x51b9a20, 0xc00d502880}, 0x692a554ac8f0f3f4?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00c70c6a0 sp=0xc00c70c5b0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x6c9a145413083e43?, {0x624a948?, 0xc03064aba0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0120d2690?}, {0x51b9a20?, 0xc00d502880?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00c70c740 sp=0xc00c70c6a0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0122773f0, {0x624a948, 0xc03064aba0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00c70c7c0 sp=0xc00c70c740 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0122773f0?}}, {0x624a948, 0xc03064aba0}, 0xc0120d2690, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00c70c898 sp=0xc00c70c7c0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x3f52900?, {0x624a948?, 0xc03064aba0?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc003f529f8?}) :1 +0x65 fp=0xc00c70c8e8 sp=0xc00c70c898 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0437cc6c0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00c70cb88 sp=0xc00c70c8e8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0437cc6c0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00c70ccc0 sp=0xc00c70cb88 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00c70d408 sp=0xc00c70ccc0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00c70dd08 sp=0xc00c70d408 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x624a948, 0xc03064aba0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1506 +0x1bc fp=0xc00c70df30 sp=0xc00c70dd08 pc=0x19ca1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c70dfe0 sp=0xc00c70df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c70dfe8 sp=0xc00c70dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 48664 [select]: runtime.gopark(0xc001122f48?, 0x2?, 0x6?, 0x0?, 0xc001122ef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001122d50 sp=0xc001122d30 pc=0x4907d6 runtime.selectgo(0xc001122f48, 0xc001122ef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001122e90 sp=0xc001122d50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0041a6d20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc001122fc8 sp=0xc001122e90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc001122fe0 sp=0xc001122fc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001122fe8 sp=0xc001122fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 284 [select]: runtime.gopark(0xc00fe0dd20?, 0x6?, 0x6?, 0x0?, 0xc00fe0dc64?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fe0dab8 sp=0xc00fe0da98 pc=0x4907d6 runtime.selectgo(0xc00fe0dd20, 0xc00fe0dc58, 0x628a748?, 0x0, 0xc00be15c00?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fe0dbf8 sp=0xc00fe0dab8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc006901e10, {0x624a948, 0xc0067c47e0}, 0xc000469520, {0x628a748?, 0xc005ec6fd0}, 0xc002a40360, 0xc0021d8210) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:363 +0x471 fp=0xc00fe0dda0 sp=0xc00fe0dbf8 pc=0x1992631 github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1({0x624a948?, 0xc0067c47e0}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:125 +0x408 fp=0xc00fe0df30 sp=0xc00fe0dda0 pc=0x19906e8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00fe0dfe0 sp=0xc00fe0df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fe0dfe8 sp=0xc00fe0dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 285 [select, 9 minutes]: runtime.gopark(0xc00697df98?, 0x2?, 0x0?, 0x0?, 0xc00697df7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00697de00 sp=0xc00697dde0 pc=0x4907d6 runtime.selectgo(0xc00697df98, 0xc00697df78, 0xc0066775c0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00697df40 sp=0xc00697de00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00666d6c0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc00697dfc8 sp=0xc00697df40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc00697dfe0 sp=0xc00697dfc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00697dfe8 sp=0xc00697dfe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 918 [select]: runtime.gopark(0xc00b60fef8?, 0x2?, 0xc0?, 0xfd?, 0xc00b60febc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dff8d18 sp=0xc00dff8cf8 pc=0x4907d6 runtime.selectgo(0xc00dff8ef8, 0xc00b60feb8, 0xc003424360?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dff8e58 sp=0xc00dff8d18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1({0x624a948, 0xc003424360}) github.com/cockroachdb/cockroach/pkg/server/node.go:760 +0x110 fp=0xc00dff8f30 sp=0xc00dff8e58 pc=0x39bd570 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00dff8fe0 sp=0xc00dff8f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dff8fe8 sp=0xc00dff8fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 287 [select]: runtime.gopark(0xc0087c7db8?, 0x3?, 0x4d?, 0x0?, 0xc0087c7d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0087c7b90 sp=0xc0087c7b70 pc=0x4907d6 runtime.selectgo(0xc0087c7db8, 0xc0087c7d2c, 0xc0067c4bd0?, 0x0, 0x46159f?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0087c7cd0 sp=0xc0087c7b90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc0067c4bd0}, 0xc00679c910, {0xc0007bed20, 0x13}, 0xc00295d740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc0087c7e68 sp=0xc0087c7cd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc0067c4bd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc0087c7f30 sp=0xc0087c7e68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0087c7fe0 sp=0xc0087c7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0087c7fe8 sp=0xc0087c7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 313 [select]: runtime.gopark(0xc0059b7f48?, 0x2?, 0x6?, 0x0?, 0xc0059b7ef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0059b7d50 sp=0xc0059b7d30 pc=0x4907d6 runtime.selectgo(0xc0059b7f48, 0xc0059b7ef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0059b7e90 sp=0xc0059b7d50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc005ba6780) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc0059b7fc8 sp=0xc0059b7e90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc0059b7fe0 sp=0xc0059b7fc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0059b7fe8 sp=0xc0059b7fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 315 [select]: runtime.gopark(0xc00470bf00?, 0x2?, 0xb0?, 0x0?, 0xc00470bed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00470bd60 sp=0xc00470bd40 pc=0x4907d6 runtime.selectgo(0xc00470bf00, 0xc00470bed0, 0x7b55a5?, 0x0, 0xc00bd6a000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00470bea0 sp=0xc00470bd60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0068cc190, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc00470bf30 sp=0xc00470bea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006205f20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc00470bf88 sp=0xc00470bf30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc00470bfe0 sp=0xc00470bf88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00470bfe8 sp=0xc00470bfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 4760 [semacquire]: runtime.gopark(0xae1b77ae1d4?, 0xfe1b44?, 0xc0?, 0xf8?, 0x20036?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0035e0058 sp=0xc0035e0038 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8c4, 0xe0?, 0x3, 0x1) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc0035e00c0 sp=0xc0035e0058 pc=0x4a17de sync.runtime_SemacquireMutex(0x1a2?, 0xa6?, 0x7f26f3ff0558?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc0035e00f0 sp=0xc0035e00c0 pc=0x4be125 sync.(*Mutex).lockSlow(0xc0005ee8c0) GOROOT/src/sync/mutex.go:171 +0x165 fp=0xc0035e0140 sp=0xc0035e00f0 pc=0x4cc8a5 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:90 sync.(*RWMutex).Lock(0xfe1b44?) GOROOT/src/sync/rwmutex.go:147 +0x36 fp=0xc0035e0170 sp=0xc0035e0140 pc=0x4cde96 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc0005ee8b0, 0x7f26f3fd0620?, 0x7f26f3fd0620?, 0x7f26f3fd0600?, 0x7f26f3fd0600) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 fp=0xc0035e0208 sp=0xc0035e0170 pc=0x10a3725 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc001f28700?, 0x7f26f3fd0620?, 0x7f26f3fd0620?, 0x20036?, 0xc0035e0310?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 fp=0xc0035e0260 sp=0xc0035e0208 pc=0x10a57a6 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004b46580, {0x13?, 0x13?}, 0x0, 0xc00c710578, 0xc00641bc10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2854 +0x60f fp=0xc0035e0360 sp=0xc0035e0260 pc=0x10cb46f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc00c710300, 0x1?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc0035e0400 sp=0xc0035e0360 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).skipForward(0xc00c710300) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1167 +0x4d fp=0xc0035e0468 sp=0xc0035e0400 pc=0x10c5eed github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).Next(0xc00c710300) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1133 +0xf9 fp=0xc0035e04c8 sp=0xc0035e0468 pc=0x10c5d19 github.com/cockroachdb/pebble.(*levelIter).Next(0xc00641cf90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:806 +0x167 fp=0xc0035e04f0 sp=0xc0035e04c8 pc=0x114c207 github.com/cockroachdb/pebble.(*mergingIter).nextEntry(0xc00641be20, 0xc007b6ba00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:555 +0x6c fp=0xc0035e0540 sp=0xc0035e04f0 pc=0x1150d8c github.com/cockroachdb/pebble.(*mergingIter).Next(0xc00641be20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:1107 +0x4e fp=0xc0035e0560 sp=0xc0035e0540 pc=0x115326e github.com/cockroachdb/pebble.(*Iterator).nextUserKey(0xc00641b900) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:663 +0x18f fp=0xc0035e05d0 sp=0xc0035e0560 pc=0x113bbaf github.com/cockroachdb/pebble.(*Iterator).NextWithLimit(0xc00641b900, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1570 +0x531 fp=0xc0035e0618 sp=0xc0035e05d0 pc=0x113fd31 github.com/cockroachdb/pebble.(*Iterator).Next(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1517 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).Next(0xc00b99cd00?) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:434 +0x48 fp=0xc0035e0648 sp=0xc0035e0618 pc=0x1863348 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).Next(0xc00b99cd00) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:819 +0x41e fp=0xc0035e0688 sp=0xc0035e0648 pc=0x180e33e github.com/cockroachdb/cockroach/pkg/storage.findSplitKeyUsingIterator({0x62b4f18, 0xc00b99cd00}, {0xc006e7d488, 0x5, 0x8}, {0xc0035e0980?, 0x12df485?, 0xc00e4ceed5?}, {0xc00e4ceed6, 0x5, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:826 +0x35f fp=0xc0035e08a8 sp=0xc0035e0688 pc=0x186615f github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).FindSplitKey(0x8?, {0xc006e7d488?, 0x0?, 0x8?}, {0xc006e7d490?, 0x0?, 0x0?}, {0xc00e4ceed6, 0x5, 0x5}, ...) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:1266 +0x92 fp=0xc0035e0990 sp=0xc0035e08a8 pc=0x1810432 github.com/cockroachdb/cockroach/pkg/storage.MVCCFindSplitKey({0xc?, 0x54b23d?}, {0x628f5a8, 0xc00268e420}, {0xc006e7d488, 0x5, 0x8}, {0xc006e7d490, 0x5, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:5464 +0x7f4 fp=0xc0035e0c48 sp=0xc0035e0990 pc=0x18382b4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).adminSplitWithDescriptor(0xc000f10800, {0x624a910, 0xc00ad69ce0}, {{{0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:331 +0x2ee fp=0xc0035e1120 sp=0xc0035e0c48 pc=0x1da1f2e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*splitQueue).processAttempt(0xc0031263c0, {0x624a910, 0xc00ad69ce0}, 0xc000f10800, {0x7f275f23e320, 0xc00a1ee240}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/split_queue.go:224 +0x445 fp=0xc0035e1590 sp=0xc0035e1120 pc=0x1e298c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*splitQueue).process(0xc0031263c0, {0x624a910, 0xc00ad69ce0}, 0x624a910?, {0x7f275f23e320?, 0xc00a1ee240?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/split_queue.go:180 +0xab fp=0xc0035e1620 sp=0xc0035e1590 pc=0x1e292ab github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x624a910, 0xc00ad69ce0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:997 +0x275 fp=0xc0035e1780 sp=0xc0035e1620 pc=0x1d77135 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x624a948?, 0xc015bf9620?}, {0xc005a74dc0, 0x20}, 0xdf8475800, 0xc0035e1920) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed fp=0xc0035e1820 sp=0xc0035e1780 pc=0x16210cd github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc000ba1180, {0x624a948, 0xc015bf95f0}, {0x6295c98, 0xc000f10800}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:956 +0x3ee fp=0xc0035e1968 sp=0xc0035e1820 pc=0x1d76cce github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplicasInPurgatory.func1.2({0x624a948, 0xc015bf95f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:1247 +0x50 fp=0xc0035e19d8 sp=0xc0035e1968 pc=0x1d78d50 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc015bf95f0}, {0x1?, 0x1?}, 0xc0035e1b40) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc0035e1a50 sp=0xc0035e19d8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplicasInPurgatory.func1(0xc000ba1180, {0x624a948, 0xc0048bb710}, 0xc002a40360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:1245 +0x547 fp=0xc0035e1be0 sp=0xc0035e1a50 pc=0x1d78c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplicasInPurgatory(0xc000ba1180, {0x624a948, 0xc0048bb710}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:1254 +0x2c fp=0xc0035e1c48 sp=0xc0035e1be0 pc=0x1d785cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).addToPurgatoryLocked.func2({0x624a948, 0xc0048bb710}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:1193 +0x27e fp=0xc0035e1f30 sp=0xc0035e1c48 pc=0x1d7827e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0035e1fe0 sp=0xc0035e1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0035e1fe8 sp=0xc0035e1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 876 [semacquire]: runtime.gopark(0x0?, 0x0?, 0x40?, 0x1d?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f40e830 sp=0xc00f40e810 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0033b2e80, 0xe8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00f40e898 sp=0xc00f40e830 pc=0x4a17de sync.runtime_Semacquire(0x1?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00f40e8c8 sp=0xc00f40e898 pc=0x4be005 sync.(*WaitGroup).Wait(0x117ca6f?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00f40e8f0 sp=0xc00f40e8c8 pc=0x4ce332 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc0003be600, 0xc0033b2d80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x8a fp=0xc00f40e920 sp=0xc00f40e8f0 pc=0x10f532a github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc0003be600, 0xc0033b2d80, 0x37?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe fp=0xc00f40e950 sp=0xc00f40e920 pc=0x10f4c7e github.com/cockroachdb/pebble.(*DB).Apply(0xc002a2cd80, 0xc0033b2d80, 0x4e4837?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:739 +0x1ce fp=0xc00f40e9d8 sp=0xc00f40e950 pc=0x111c86e github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:1049 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc040031170?, 0x1?) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:546 +0x3c fp=0xc00f40ea00 sp=0xc00f40e9d8 pc=0x185d37c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:983 +0x188a fp=0xc00f40f4b0 sp=0xc00f40ea00 pc=0x1ddf7aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:664 +0x17b fp=0xc00f40f9b0 sp=0xc00f40f4b0 pc=0x1dddd1b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc00679ea80, 0x624a948?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:641 +0xb8 fp=0xc00f40fe88 sp=0xc00f40f9b0 pc=0x1e48038 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1350}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x264 fp=0xc00f40ff08 sp=0xc00f40fe88 pc=0x1e26f24 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1350?}) :1 +0x39 fp=0xc00f40ff30 sp=0xc00f40ff08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f40ffe0 sp=0xc00f40ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f40ffe8 sp=0xc00f40ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 273 [select]: runtime.gopark(0xc0000ac8d8?, 0x2?, 0xb0?, 0xc7?, 0xc0000ac8a4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0000ac728 sp=0xc0000ac708 pc=0x4907d6 runtime.selectgo(0xc0000ac8d8, 0xc0000ac8a0, 0x46159f?, 0x0, 0xc0231de000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0000ac868 sp=0xc0000ac728 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc005923680, {0xc0061e6d78, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc0000ac908 sp=0xc0000ac868 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc005923680, {0xc0061e6d78?, 0xc0061fc150?, 0xc0000ac950?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc0000ac948 sp=0xc0000ac908 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc006709a10, {0xc0061e6d78?, 0xc0000ac9c8?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc0000ac990 sp=0xc0000ac948 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc006709a10}, {0xc0061e6d78, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc0000ac9d8 sp=0xc0000ac990 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006735320, {0xc0061e6d78, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc0000aca18 sp=0xc0000ac9d8 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc0061e6d68, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc0000aca98 sp=0xc0000aca18 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x4908c0?, 0xc006735320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc0000acb30 sp=0xc0000aca98 pc=0xc19546 google.golang.org/grpc.recv(0x0?, {0x7f275f301038, 0xc0000af170}, 0x0?, {0x0?, 0x0?}, {0x5125d60, 0xc0078f2280}, 0xc0134d3f80?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc0000acba8 sp=0xc0000acb30 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc005a12160, {0x5125d60?, 0xc0078f2280}, 0xc0067c4780?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc0000accb0 sp=0xc0000acba8 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0000acce0 sp=0xc0000accb0 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0067350e0, 0xc0000acdb8, 0xc0000acd88) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc0000acd38 sp=0xc0000acce0 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0067350e0, {0x5125d60?, 0xc0078f2280?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0000acde8 sp=0xc0000acd38 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc002fdab80, {0x5125d60?, 0xc0078f2280?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc0000ace30 sp=0xc0000acde8 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc005ec6fd0) github.com/cockroachdb/cockroach/pkg/gossip/bazel-out/k8-opt/bin/pkg/gossip/gossip_go_proto_/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:636 +0x4c fp=0xc0000ace60 sp=0xc0000ace30 pc=0x19813ec github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0xc0039e89c0?, {0x628a748, 0xc005ec6fd0}, 0xc006901e10, {0x624a948, 0xc006709b60}, 0xc0007fe0c0?) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:317 +0x66 fp=0xc0000aceb0 sp=0xc0000ace60 pc=0x1992b86 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2({0x624a948?, 0xc006709b60?}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:331 +0x91 fp=0xc0000acf30 sp=0xc0000aceb0 pc=0x1992ab1 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0000acfe0 sp=0xc0000acf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000acfe8 sp=0xc0000acfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49139 [runnable]: runtime.Gosched(...) GOROOT/src/runtime/proc.go:318 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00844ed20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:570 +0x1f6 fp=0xc011577f88 sp=0xc011577f30 pc=0xbd7156 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc011577fe0 sp=0xc011577f88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011577fe8 sp=0xc011577fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 484647 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x315?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ee6af58 sp=0xc00ee6af38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f3003a8?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00ee6af90 sp=0xc00ee6af58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f3002c0, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00ee6afb0 sp=0xc00ee6af90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc03f820c80?, 0xc005c84000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00ee6afd8 sp=0xc00ee6afb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc03f820c80, {0xc005c84000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00ee6b058 sp=0xc00ee6afd8 pc=0x535efa net.(*netFD).Read(0xc03f820c80, {0xc005c84000?, 0x7b5a68?, 0xc03f820c80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00ee6b0a0 sp=0xc00ee6b058 pc=0x7a20c9 net.(*conn).Read(0xc0134d9a60, {0xc005c84000?, 0x164290f?, 0xc0134d9a60?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00ee6b0e8 sp=0xc00ee6b0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00770bce0, {0xc005c84000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00ee6b130 sp=0xc00ee6b0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc013d82780, {0xc005c84000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00ee6b1d0 sp=0xc00ee6b130 pc=0x370f896 bufio.(*Reader).fill(0xc00312c890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00ee6b208 sp=0xc00ee6b1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00312c890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00ee6b228 sp=0xc00ee6b208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00312ce68?, {0x62295c8, 0xc00312c890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00ee6b260 sp=0xc00ee6b228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00312c800, 0xc0138fc590, 0xc00ee6b442, 0xc00ee6b444, {0x623b718, 0xc02ea35290}, 0xc0059fa000, 0xc001f25f04, 0xc00ee6b440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00ee6b398 sp=0xc00ee6b260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00312c800, {0x624a948, 0xc03cec9ef0}, 0xc0138fc580, 0xc0059fa000, 0xc03cec9e30, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00ee6b768 sp=0xc00ee6b398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc03cec9ef0}, {0x628a538, 0xc00770bce0}, {{{0xc003f73038, 0x4}}, 0x1, 0xc03cec9e60, 0xc03cec9e90, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00ee6b8a8 sp=0xc00ee6b768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc03cec9ef0}, {0x628a538?, 0xc00770bce0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00ee6be20 sp=0xc00ee6b8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc03cec9ce0?}, {0x628a538, 0xc00770bce0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00ee6beb0 sp=0xc00ee6be20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc03cec9ce0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00ee6bf30 sp=0xc00ee6beb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ee6bfe0 sp=0xc00ee6bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ee6bfe8 sp=0xc00ee6bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1257 [select]: runtime.gopark(0xc00ca63ef8?, 0x2?, 0x0?, 0x0?, 0xc00ca63e6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0049ebcc8 sp=0xc0049ebca8 pc=0x4907d6 runtime.selectgo(0xc0049ebef8, 0xc00ca63e68, 0xc000c50000?, 0x0, 0x56839a8?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0049ebe08 sp=0xc0049ebcc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1({0x624a948?, 0xc0034e8630?}) github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:443 +0x2a5 fp=0xc0049ebf30 sp=0xc0049ebe08 pc=0x228d725 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0049ebfe0 sp=0xc0049ebf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0049ebfe8 sp=0xc0049ebfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 910 [select, 9 minutes]: runtime.gopark(0xc005a60f98?, 0x2?, 0x0?, 0x0?, 0xc005a60f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a60e00 sp=0xc005a60de0 pc=0x4907d6 runtime.selectgo(0xc005a60f98, 0xc005a60f78, 0xc00337e900?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005a60f40 sp=0xc005a60e00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc005bc75c0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc005a60fc8 sp=0xc005a60f40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc005a60fe0 sp=0xc005a60fc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a60fe8 sp=0xc005a60fe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 3735 [select]: runtime.gopark(0xc001834e38?, 0x3?, 0x0?, 0x0?, 0xc001834dea?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001834c40 sp=0xc001834c20 pc=0x4907d6 runtime.selectgo(0xc001834e38, 0xc001834de4, 0xc007814890?, 0x0, 0x4618e7?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001834d80 sp=0xc001834c40 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00477f6c0, {0x624a8a0, 0xc001e2fb40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:322 +0x265 fp=0xc001834e78 sp=0xc001834d80 pc=0x1c8b565 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00477f6c0, {0x624a948, 0xc002bc6390}, 0x8?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:347 +0xc5 fp=0xc001834eb0 sp=0xc001834e78 pc=0x1c8b7c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x624a948?, 0xc002bc6390?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:296 +0x4c fp=0xc001834f30 sp=0xc001834eb0 pc=0x1c86a8c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001834fe0 sp=0xc001834f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001834fe8 sp=0xc001834fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 982 [select]: runtime.gopark(0xc0036baf48?, 0x2?, 0x6?, 0x0?, 0xc0036baef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0036bad50 sp=0xc0036bad30 pc=0x4907d6 runtime.selectgo(0xc0036baf48, 0xc0036baef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0036bae90 sp=0xc0036bad50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0032e2000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc0036bafc8 sp=0xc0036bae90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc0036bafe0 sp=0xc0036bafc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0036bafe8 sp=0xc0036bafe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 896 [select, 9 minutes]: runtime.gopark(0xc0032db7b0?, 0x2?, 0x2d?, 0xe4?, 0xc0032db794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032db618 sp=0xc0032db5f8 pc=0x4907d6 runtime.selectgo(0xc0032db7b0, 0xc0032db790, 0xc0032db750?, 0x0, 0xc0032db7d0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032db758 sp=0xc0032db618 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0032db7e0 sp=0xc0032db758 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032db7e8 sp=0xc0032db7e0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 921 [select, 9 minutes]: runtime.gopark(0xc0042356b0?, 0x3?, 0x1?, 0x30?, 0xc00423566a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0042354b8 sp=0xc004235498 pc=0x4907d6 runtime.selectgo(0xc0042356b0, 0xc004235664, 0x1?, 0x0, 0xc005976e18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0042355f8 sp=0xc0042354b8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc000920000, 0xc002141ab0, 0xc0057eb850, 0x568e068, 0xc00266da40, 0x0) github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:193 +0x152 fp=0xc0042356f0 sp=0xc0042355f8 pc=0x398e4b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func3({0xc000ca1880, 0xc007ec8f50}) github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:268 +0x49 fp=0xc004235730 sp=0xc0042356f0 pc=0x398eb09 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0042357e0 sp=0xc004235730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0042357e8 sp=0xc0042357e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 897 [select, 9 minutes]: runtime.gopark(0xc0011ea7b0?, 0x2?, 0x32?, 0x11?, 0xc0011ea79c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0011ea628 sp=0xc0011ea608 pc=0x4907d6 runtime.selectgo(0xc0011ea7b0, 0xc0011ea798, 0xc002b7d1e0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0011ea768 sp=0xc0011ea628 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0011ea7e0 sp=0xc0011ea768 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0011ea7e8 sp=0xc0011ea7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 909 [select]: runtime.gopark(0xc00a9c7950?, 0x4?, 0xc8?, 0x77?, 0xc00a9c78a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a9c7700 sp=0xc00a9c76e0 pc=0x4907d6 runtime.selectgo(0xc00a9c7950, 0xc00a9c7898, 0xc00eb72ea0?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a9c7840 sp=0xc00a9c7700 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000c62700, 0xc00337e440, {0x628a900, 0xc003451600}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:434 +0x345 fp=0xc00a9c7cd8 sp=0xc00a9c7840 pc=0x1d83c25 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x624a948, 0xc0034ad4d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:601 +0x367 fp=0xc00a9c7e18 sp=0xc00a9c7cd8 pc=0x1d852e7 runtime/pprof.Do({0x624a948?, 0xc0034ad470?}, {{0xc0043c6a40?, 0xc0011eb6b8?, 0xc448e6?}}, 0xc0043c6a20) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00a9c7e88 sp=0xc00a9c7e18 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x624a948, 0xc0034ad470}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:607 +0x1c8 fp=0xc00a9c7f30 sp=0xc00a9c7e88 pc=0x1d84f28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a9c7fe0 sp=0xc00a9c7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a9c7fe8 sp=0xc00a9c7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 919 [select, 9 minutes]: runtime.gopark(0xc0059837b0?, 0x2?, 0x1?, 0x30?, 0xc00598375c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0059835b0 sp=0xc005983590 pc=0x4907d6 runtime.selectgo(0xc0059837b0, 0xc005983758, 0x1058490?, 0x0, 0xc001a44f00?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0059836f0 sp=0xc0059835b0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:338 +0xd5 fp=0xc0059837e0 sp=0xc0059836f0 pc=0x1cab0b5 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0059837e8 sp=0xc0059837e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:331 +0xb0 goroutine 877 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196ee436d?, 0x7fecdd71bf?, 0x0?, 0x0?, 0xc1116e81974c74ca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0051ade08 sp=0xc0051adde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb2a) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0051ade50 sp=0xc0051ade08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0051ade88 sp=0xc0051ade50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a13b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc0051adf08 sp=0xc0051ade88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a13b0?}) :1 +0x39 fp=0xc0051adf30 sp=0xc0051adf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0051adfe0 sp=0xc0051adf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0051adfe8 sp=0xc0051adfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 878 [semacquire]: runtime.gopark(0x0?, 0x0?, 0xa0?, 0x5?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00539a830 sp=0xc00539a810 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0051cdc00, 0xe8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00539a898 sp=0xc00539a830 pc=0x4a17de sync.runtime_Semacquire(0x1?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00539a8c8 sp=0xc00539a898 pc=0x4be005 sync.(*WaitGroup).Wait(0x117ca6f?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00539a8f0 sp=0xc00539a8c8 pc=0x4ce332 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc0003be600, 0xc0051cdb00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x8a fp=0xc00539a920 sp=0xc00539a8f0 pc=0x10f532a github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc0003be600, 0xc0051cdb00, 0x38?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe fp=0xc00539a950 sp=0xc00539a920 pc=0x10f4c7e github.com/cockroachdb/pebble.(*DB).Apply(0xc002a2cd80, 0xc0051cdb00, 0x4e4837?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:739 +0x1ce fp=0xc00539a9d8 sp=0xc00539a950 pc=0x111c86e github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:1049 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc040030cc0?, 0x1?) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:546 +0x3c fp=0xc00539aa00 sp=0xc00539a9d8 pc=0x185d37c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:983 +0x188a fp=0xc00539b4b0 sp=0xc00539aa00 pc=0x1ddf7aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:664 +0x17b fp=0xc00539b9b0 sp=0xc00539b4b0 pc=0x1dddd1b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc00679ea80, 0x624a948?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:641 +0xb8 fp=0xc00539be88 sp=0xc00539b9b0 pc=0x1e48038 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1410}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x264 fp=0xc00539bf08 sp=0xc00539be88 pc=0x1e26f24 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1410?}) :1 +0x39 fp=0xc00539bf30 sp=0xc00539bf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00539bfe0 sp=0xc00539bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00539bfe8 sp=0xc00539bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 879 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196fffae1?, 0x7fecef292e?, 0x0?, 0x0?, 0xc1116e81974ef82b?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007589e08 sp=0xc007589de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb33) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc007589e50 sp=0xc007589e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc007589e88 sp=0xc007589e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1470}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc007589f08 sp=0xc007589e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1470?}) :1 +0x39 fp=0xc007589f30 sp=0xc007589f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc007589fe0 sp=0xc007589f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007589fe8 sp=0xc007589fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 880 [sync.Cond.Wait]: runtime.gopark(0xc011c24060?, 0x7fec272718?, 0x0?, 0x0?, 0xc1116e81968dbf92?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0039dde08 sp=0xc0039ddde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb23) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0039dde50 sp=0xc0039dde08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0039dde88 sp=0xc0039dde50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a14d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc0039ddf08 sp=0xc0039dde88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a14d0?}) :1 +0x39 fp=0xc0039ddf30 sp=0xc0039ddf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0039ddfe0 sp=0xc0039ddf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0039ddfe8 sp=0xc0039ddfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 881 [semacquire]: runtime.gopark(0x0?, 0x0?, 0x20?, 0xe4?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0108e8830 sp=0xc0108e8810 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0150d07c0, 0xe8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc0108e8898 sp=0xc0108e8830 pc=0x4a17de sync.runtime_Semacquire(0x2?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc0108e88c8 sp=0xc0108e8898 pc=0x4be005 sync.(*WaitGroup).Wait(0x117ca6f?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc0108e88f0 sp=0xc0108e88c8 pc=0x4ce332 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc0003be600, 0xc0150d06c0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x8a fp=0xc0108e8920 sp=0xc0108e88f0 pc=0x10f532a github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc0003be600, 0xc0150d06c0, 0xa5?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe fp=0xc0108e8950 sp=0xc0108e8920 pc=0x10f4c7e github.com/cockroachdb/pebble.(*DB).Apply(0xc002a2cd80, 0xc0150d06c0, 0x4e4837?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:739 +0x1ce fp=0xc0108e89d8 sp=0xc0108e8950 pc=0x111c86e github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:1049 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc00cdb4600?, 0x2?) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:546 +0x3c fp=0xc0108e8a00 sp=0xc0108e89d8 pc=0x185d37c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:983 +0x188a fp=0xc0108e94b0 sp=0xc0108e8a00 pc=0x1ddf7aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:664 +0x17b fp=0xc0108e99b0 sp=0xc0108e94b0 pc=0x1dddd1b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc00679ea80, 0xc006795590?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:641 +0xb8 fp=0xc0108e9e88 sp=0xc0108e99b0 pc=0x1e48038 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1530}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x264 fp=0xc0108e9f08 sp=0xc0108e9e88 pc=0x1e26f24 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1530?}) :1 +0x39 fp=0xc0108e9f30 sp=0xc0108e9f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0108e9fe0 sp=0xc0108e9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0108e9fe8 sp=0xc0108e9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 930 [semacquire]: runtime.gopark(0x0?, 0x0?, 0x40?, 0x8b?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003b08830 sp=0xc003b08810 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc013a47c00, 0xe8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc003b08898 sp=0xc003b08830 pc=0x4a17de sync.runtime_Semacquire(0x1?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc003b088c8 sp=0xc003b08898 pc=0x4be005 sync.(*WaitGroup).Wait(0x117ca6f?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc003b088f0 sp=0xc003b088c8 pc=0x4ce332 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc0003be600, 0xc013a47b00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x8a fp=0xc003b08920 sp=0xc003b088f0 pc=0x10f532a github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc0003be600, 0xc013a47b00, 0xe6?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe fp=0xc003b08950 sp=0xc003b08920 pc=0x10f4c7e github.com/cockroachdb/pebble.(*DB).Apply(0xc002a2cd80, 0xc013a47b00, 0x4e4837?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:739 +0x1ce fp=0xc003b089d8 sp=0xc003b08950 pc=0x111c86e github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:1049 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc005ee6ea0?, 0x1?) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:546 +0x3c fp=0xc003b08a00 sp=0xc003b089d8 pc=0x185d37c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:983 +0x188a fp=0xc003b094b0 sp=0xc003b08a00 pc=0x1ddf7aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:664 +0x17b fp=0xc003b099b0 sp=0xc003b094b0 pc=0x1dddd1b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc00679ea80, 0xc006795590?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:641 +0xb8 fp=0xc003b09e88 sp=0xc003b099b0 pc=0x1e48038 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1590}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x264 fp=0xc003b09f08 sp=0xc003b09e88 pc=0x1e26f24 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1590?}) :1 +0x39 fp=0xc003b09f30 sp=0xc003b09f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003b09fe0 sp=0xc003b09f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003b09fe8 sp=0xc003b09fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 931 [sync.Cond.Wait]: runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aede08 sp=0xc004aedde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb28) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc004aede50 sp=0xc004aede08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc004aede88 sp=0xc004aede50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a15f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc004aedf08 sp=0xc004aede88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a15f0?}) :1 +0x39 fp=0xc004aedf30 sp=0xc004aedf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc004aedfe0 sp=0xc004aedf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aedfe8 sp=0xc004aedfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 932 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196f93001?, 0x7fece85e51?, 0x0?, 0x0?, 0xc1116e81974e1851?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0063cde08 sp=0xc0063cdde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb2f) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0063cde50 sp=0xc0063cde08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0063cde88 sp=0xc0063cde50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1650}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc0063cdf08 sp=0xc0063cde88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1650?}) :1 +0x39 fp=0xc0063cdf30 sp=0xc0063cdf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0063cdfe0 sp=0xc0063cdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0063cdfe8 sp=0xc0063cdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 933 [sync.Cond.Wait]: runtime.gopark(0xc00c7f0f60?, 0x7feb6a5c06?, 0x0?, 0x0?, 0xc1116e8196992924?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005f37e08 sp=0xc005f37de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb24) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc005f37e50 sp=0xc005f37e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc005f37e88 sp=0xc005f37e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a16b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc005f37f08 sp=0xc005f37e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a16b0?}) :1 +0x39 fp=0xc005f37f30 sp=0xc005f37f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005f37fe0 sp=0xc005f37f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005f37fe8 sp=0xc005f37fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 934 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196f887c2?, 0x7fece7b614?, 0x0?, 0x0?, 0xc1116e81974de08e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002bd1e08 sp=0xc002bd1de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb2e) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc002bd1e50 sp=0xc002bd1e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc002bd1e88 sp=0xc002bd1e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1710}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc002bd1f08 sp=0xc002bd1e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1710?}) :1 +0x39 fp=0xc002bd1f30 sp=0xc002bd1f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002bd1fe0 sp=0xc002bd1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002bd1fe8 sp=0xc002bd1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 935 [sync.Cond.Wait]: runtime.gopark(0xc011cecb40?, 0x7fec17b3e0?, 0x0?, 0x0?, 0xc1116e819688f2f7?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012709e08 sp=0xc012709de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb3a) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc012709e50 sp=0xc012709e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc012709e88 sp=0xc012709e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1770}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc012709f08 sp=0xc012709e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1770?}) :1 +0x39 fp=0xc012709f30 sp=0xc012709f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012709fe0 sp=0xc012709f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012709fe8 sp=0xc012709fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 936 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196fa6173?, 0x7fece98fc5?, 0x0?, 0x0?, 0xc1116e81974e84fc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00658be08 sp=0xc00658bde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb31) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00658be50 sp=0xc00658be08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00658be88 sp=0xc00658be50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a17d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc00658bf08 sp=0xc00658be88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a17d0?}) :1 +0x39 fp=0xc00658bf30 sp=0xc00658bf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00658bfe0 sp=0xc00658bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00658bfe8 sp=0xc00658bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 937 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196f9ea06?, 0x7fece91858?, 0x0?, 0x0?, 0xc1116e81974e4ef7?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003e81e08 sp=0xc003e81de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb30) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc003e81e50 sp=0xc003e81e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc003e81e88 sp=0xc003e81e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1830}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc003e81f08 sp=0xc003e81e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1830?}) :1 +0x39 fp=0xc003e81f30 sp=0xc003e81f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003e81fe0 sp=0xc003e81f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003e81fe8 sp=0xc003e81fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 938 [sync.Cond.Wait]: runtime.gopark(0xc00d41de60?, 0x7fec23886d?, 0x0?, 0x0?, 0xc1116e81968d9764?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006167e08 sp=0xc006167de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb22) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc006167e50 sp=0xc006167e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc006167e88 sp=0xc006167e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1890}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc006167f08 sp=0xc006167e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1890?}) :1 +0x39 fp=0xc006167f30 sp=0xc006167f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006167fe0 sp=0xc006167f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006167fe8 sp=0xc006167fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 939 [semacquire]: runtime.gopark(0x0?, 0x0?, 0xa0?, 0x7a?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003f96830 sp=0xc003f96810 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc013f99ae0, 0xe8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc003f96898 sp=0xc003f96830 pc=0x4a17de sync.runtime_Semacquire(0x2?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc003f968c8 sp=0xc003f96898 pc=0x4be005 sync.(*WaitGroup).Wait(0x117ca6f?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc003f968f0 sp=0xc003f968c8 pc=0x4ce332 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc0003be600, 0xc013f999e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x8a fp=0xc003f96920 sp=0xc003f968f0 pc=0x10f532a github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc0003be600, 0xc013f999e0, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe fp=0xc003f96950 sp=0xc003f96920 pc=0x10f4c7e github.com/cockroachdb/pebble.(*DB).Apply(0xc002a2cd80, 0xc013f999e0, 0x4e4837?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:739 +0x1ce fp=0xc003f969d8 sp=0xc003f96950 pc=0x111c86e github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:1049 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc?, 0x2?) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:546 +0x3c fp=0xc003f96a00 sp=0xc003f969d8 pc=0x185d37c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:983 +0x188a fp=0xc003f974b0 sp=0xc003f96a00 pc=0x1ddf7aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:664 +0x17b fp=0xc003f979b0 sp=0xc003f974b0 pc=0x1dddd1b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc00679ea80, 0x624a948?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:641 +0xb8 fp=0xc003f97e88 sp=0xc003f979b0 pc=0x1e48038 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a18f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x264 fp=0xc003f97f08 sp=0xc003f97e88 pc=0x1e26f24 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a18f0?}) :1 +0x39 fp=0xc003f97f30 sp=0xc003f97f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003f97fe0 sp=0xc003f97f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003f97fe8 sp=0xc003f97fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 940 [sync.Cond.Wait]: runtime.gopark(0xc011240d20?, 0x7feba8f7fc?, 0x0?, 0x0?, 0xc1116e81969b9e3f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01218be08 sp=0xc01218bde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb36) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc01218be50 sp=0xc01218be08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc01218be88 sp=0xc01218be50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1950}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc01218bf08 sp=0xc01218be88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1950?}) :1 +0x39 fp=0xc01218bf30 sp=0xc01218bf08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01218bfe0 sp=0xc01218bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01218bfe8 sp=0xc01218bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 941 [sync.Cond.Wait]: runtime.gopark(0xc1116e819733aa4f?, 0x7fed22d89c?, 0x0?, 0x0?, 0xc1116e81975056b4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ec05e08 sp=0xc00ec05de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb35) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00ec05e50 sp=0xc00ec05e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00ec05e88 sp=0xc00ec05e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a19b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc00ec05f08 sp=0xc00ec05e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a19b0?}) :1 +0x39 fp=0xc00ec05f30 sp=0xc00ec05f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ec05fe0 sp=0xc00ec05f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ec05fe8 sp=0xc00ec05fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 942 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196f75f76?, 0x7fece68dc6?, 0x0?, 0x0?, 0xc1116e81974d5a30?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005053e08 sp=0xc005053de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb2c) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc005053e50 sp=0xc005053e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc005053e88 sp=0xc005053e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1a10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc005053f08 sp=0xc005053e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1a10?}) :1 +0x39 fp=0xc005053f30 sp=0xc005053f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005053fe0 sp=0xc005053f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005053fe8 sp=0xc005053fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 943 [sync.Cond.Wait]: runtime.gopark(0xc1116e8197fb382f?, 0x7fedea6694?, 0x0?, 0x0?, 0xc1116e8197fb64e3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002239e08 sp=0xc002239de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb38) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc002239e50 sp=0xc002239e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc002239e88 sp=0xc002239e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1a70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc002239f08 sp=0xc002239e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1a70?}) :1 +0x39 fp=0xc002239f30 sp=0xc002239f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002239fe0 sp=0xc002239f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002239fe8 sp=0xc002239fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 944 [sync.Cond.Wait]: runtime.gopark(0xc004d58120?, 0x7feb71e0e3?, 0x0?, 0x0?, 0xc1116e819699d862?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003be5e08 sp=0xc003be5de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb25) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc003be5e50 sp=0xc003be5e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc003be5e88 sp=0xc003be5e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1ad0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc003be5f08 sp=0xc003be5e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1ad0?}) :1 +0x39 fp=0xc003be5f30 sp=0xc003be5f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003be5fe0 sp=0xc003be5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003be5fe8 sp=0xc003be5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 945 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196fd563f?, 0x7fecec848b?, 0x0?, 0x0?, 0xc1116e81974ebb2e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00151de08 sp=0xc00151dde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb32) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00151de50 sp=0xc00151de08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00151de88 sp=0xc00151de50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc00151df08 sp=0xc00151de88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1b30?}) :1 +0x39 fp=0xc00151df30 sp=0xc00151df08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00151dfe0 sp=0xc00151df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00151dfe8 sp=0xc00151dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 946 [semacquire]: runtime.gopark(0x0?, 0x0?, 0x0?, 0xb2?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004a5c830 sp=0xc004a5c810 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc00e724100, 0xe8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc004a5c898 sp=0xc004a5c830 pc=0x4a17de sync.runtime_Semacquire(0x1?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc004a5c8c8 sp=0xc004a5c898 pc=0x4be005 sync.(*WaitGroup).Wait(0x117ca6f?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc004a5c8f0 sp=0xc004a5c8c8 pc=0x4ce332 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc0003be600, 0xc00e724000) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x8a fp=0xc004a5c920 sp=0xc004a5c8f0 pc=0x10f532a github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc0003be600, 0xc00e724000, 0x68?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe fp=0xc004a5c950 sp=0xc004a5c920 pc=0x10f4c7e github.com/cockroachdb/pebble.(*DB).Apply(0xc002a2cd80, 0xc00e724000, 0x4e4837?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:739 +0x1ce fp=0xc004a5c9d8 sp=0xc004a5c950 pc=0x111c86e github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:1049 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc040030f60?, 0x1?) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:546 +0x3c fp=0xc004a5ca00 sp=0xc004a5c9d8 pc=0x185d37c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:983 +0x188a fp=0xc004a5d4b0 sp=0xc004a5ca00 pc=0x1ddf7aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:664 +0x17b fp=0xc004a5d9b0 sp=0xc004a5d4b0 pc=0x1dddd1b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc00679ea80, 0x624a948?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:641 +0xb8 fp=0xc004a5de88 sp=0xc004a5d9b0 pc=0x1e48038 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1b90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x264 fp=0xc004a5df08 sp=0xc004a5de88 pc=0x1e26f24 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1b90?}) :1 +0x39 fp=0xc004a5df30 sp=0xc004a5df08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc004a5dfe0 sp=0xc004a5df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004a5dfe8 sp=0xc004a5dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 947 [sync.Cond.Wait]: runtime.gopark(0xc1116e819700c95e?, 0x7feceff7b4?, 0x0?, 0x0?, 0xc1116e81974b5c9e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012899e08 sp=0xc012899de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb29) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc012899e50 sp=0xc012899e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc012899e88 sp=0xc012899e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1bf0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc012899f08 sp=0xc012899e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1bf0?}) :1 +0x39 fp=0xc012899f30 sp=0xc012899f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012899fe0 sp=0xc012899f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012899fe8 sp=0xc012899fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 948 [sync.Cond.Wait]: runtime.gopark(0xc0048c0de0?, 0x7fec1f0a42?, 0x0?, 0x0?, 0xc1116e8196892f39?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0065d7e08 sp=0xc0065d7de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb3b) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0065d7e50 sp=0xc0065d7e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0065d7e88 sp=0xc0065d7e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc0065d7f08 sp=0xc0065d7e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1c50?}) :1 +0x39 fp=0xc0065d7f30 sp=0xc0065d7f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0065d7fe0 sp=0xc0065d7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0065d7fe8 sp=0xc0065d7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 949 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196f7b037?, 0x7fece6de89?, 0x0?, 0x0?, 0xc1116e81974d938f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003957e08 sp=0xc003957de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb2d) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc003957e50 sp=0xc003957e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc003957e88 sp=0xc003957e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1cb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc003957f08 sp=0xc003957e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1cb0?}) :1 +0x39 fp=0xc003957f30 sp=0xc003957f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003957fe0 sp=0xc003957f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003957fe8 sp=0xc003957fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 950 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196f35eba?, 0x7fece28d0d?, 0x0?, 0x0?, 0xc1116e81974cf512?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002c6de08 sp=0xc002c6dde8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb2b) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc002c6de50 sp=0xc002c6de08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc002c6de88 sp=0xc002c6de50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1d10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc002c6df08 sp=0xc002c6de88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1d10?}) :1 +0x39 fp=0xc002c6df30 sp=0xc002c6df08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002c6dfe0 sp=0xc002c6df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002c6dfe8 sp=0xc002c6dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 951 [sync.Cond.Wait]: runtime.gopark(0xc1116e81972604b2?, 0x7fed153316?, 0x0?, 0x0?, 0xc1116e8197261b34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002e15e08 sp=0xc002e15de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb27) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc002e15e50 sp=0xc002e15e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc002e15e88 sp=0xc002e15e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1d70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc002e15f08 sp=0xc002e15e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1d70?}) :1 +0x39 fp=0xc002e15f30 sp=0xc002e15f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002e15fe0 sp=0xc002e15f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002e15fe8 sp=0xc002e15fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 952 [sync.Cond.Wait]: runtime.gopark(0xc1116e81975a652e?, 0x7fed49937a?, 0x0?, 0x0?, 0xc1116e8197ecfd7b?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0105b1e08 sp=0xc0105b1de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb37) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0105b1e50 sp=0xc0105b1e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0105b1e88 sp=0xc0105b1e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1dd0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc0105b1f08 sp=0xc0105b1e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1dd0?}) :1 +0x39 fp=0xc0105b1f30 sp=0xc0105b1f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0105b1fe0 sp=0xc0105b1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0105b1fe8 sp=0xc0105b1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 953 [sync.Cond.Wait]: runtime.gopark(0xc02b2584e0?, 0x7fec2fc1af?, 0x0?, 0x0?, 0xc1116e81967b7aa6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003ab3e08 sp=0xc003ab3de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb39) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc003ab3e50 sp=0xc003ab3e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc003ab3e88 sp=0xc003ab3e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1e30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc003ab3f08 sp=0xc003ab3e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1e30?}) :1 +0x39 fp=0xc003ab3f30 sp=0xc003ab3f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003ab3fe0 sp=0xc003ab3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003ab3fe8 sp=0xc003ab3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 954 [sync.Cond.Wait]: runtime.gopark(0xc1116e8197007703?, 0x7fecefa551?, 0x0?, 0x0?, 0xc1116e81974f5468?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003673e08 sp=0xc003673de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb34) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc003673e50 sp=0xc003673e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc003673e88 sp=0xc003673e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1e90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc003673f08 sp=0xc003673e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1e90?}) :1 +0x39 fp=0xc003673f30 sp=0xc003673f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003673fe0 sp=0xc003673f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003673fe8 sp=0xc003673fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 955 [sync.Cond.Wait]: runtime.gopark(0xc1116e8196fe2c10?, 0x7feced5a7e?, 0x0?, 0x0?, 0xc1116e819700e5e7?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006405e08 sp=0xc006405de8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00666d050, 0x9bb26) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc006405e50 sp=0xc006405e08 pc=0x4be30c sync.(*Cond).Wait(0x4b588c0?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc006405e88 sp=0xc006405e50 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0005591d0, {0x624a948, 0xc0032a1ef0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:275 +0x2ff fp=0xc006405f08 sp=0xc006405e88 pc=0x1e26fbf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker-fm({0x624a948?, 0xc0032a1ef0?}) :1 +0x39 fp=0xc006405f30 sp=0xc006405f08 pc=0x1e852b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006405fe0 sp=0xc006405f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006405fe8 sp=0xc006405fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 956 [semacquire, 9 minutes]: runtime.gopark(0x11?, 0x11?, 0x0?, 0xaf?, 0x1?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032dfe30 sp=0xc0032dfe10 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005592b8, 0xb8?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc0032dfe98 sp=0xc0032dfe30 pc=0x4a17de sync.runtime_Semacquire(0x8f?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc0032dfec8 sp=0xc0032dfe98 pc=0x4be005 sync.(*WaitGroup).Wait(0xc001b83948?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc0032dfef0 sp=0xc0032dfec8 pc=0x4ce332 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(0xc45d72?, {0x3bc?, 0x0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:236 +0x25 fp=0xc0032dff08 sp=0xc0032dfef0 pc=0x1e26a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait-fm({0x624a948?, 0xc0032a1f50?}) :1 +0x39 fp=0xc0032dff30 sp=0xc0032dff08 pc=0x1e854f9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0032dffe0 sp=0xc0032dff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032dffe8 sp=0xc0032dffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 957 [select]: runtime.gopark(0xc006a45e70?, 0x2?, 0x0?, 0x0?, 0xc006a45e3c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a45c90 sp=0xc006a45c70 pc=0x4907d6 runtime.selectgo(0xc006a45e70, 0xc006a45e38, 0xc006a45e90?, 0x0, 0x10?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006a45dd0 sp=0xc006a45c90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc00679ea80, {0x3bd?, 0x56a46a?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:749 +0x150 fp=0xc006a45f08 sp=0xc006a45dd0 pc=0x1e48ff0 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop-fm({0x624a948?, 0xc0032a1fb0?}) :1 +0x39 fp=0xc006a45f30 sp=0xc006a45f08 pc=0x1e85579 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006a45fe0 sp=0xc006a45f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a45fe8 sp=0xc006a45fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 958 [select]: runtime.gopark(0xc006091ed0?, 0x2?, 0xec?, 0xa0?, 0xc006091eb4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006091d38 sp=0xc006091d18 pc=0x4907d6 runtime.selectgo(0xc006091ed0, 0xc006091eb0, 0xc0032ba030?, 0x0, 0xb?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006091e78 sp=0xc006091d38 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc00679ea80, {0x624a948, 0xc0032ba030}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:829 +0x105 fp=0xc006091f08 sp=0xc006091e78 pc=0x1e497c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop-fm({0x624a948?, 0xc0032ba030?}) :1 +0x39 fp=0xc006091f30 sp=0xc006091f08 pc=0x1e855f9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006091fe0 sp=0xc006091f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006091fe8 sp=0xc006091fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 959 [select, 3 minutes]: runtime.gopark(0xc0032ed700?, 0x2?, 0x0?, 0x0?, 0xc0032ed6e4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032ed568 sp=0xc0032ed548 pc=0x4907d6 runtime.selectgo(0xc0032ed700, 0xc0032ed6e0, 0xc0032ed700?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032ed6a8 sp=0xc0032ed568 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func3({0xc00295ea80, 0x0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2079 +0xb9 fp=0xc0032ed730 sp=0xc0032ed6a8 pc=0x1e32219 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0032ed7e0 sp=0xc0032ed730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032ed7e8 sp=0xc0032ed7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 960 [select]: runtime.gopark(0xc00d885dd0?, 0x2?, 0x0?, 0x66?, 0xc00d885d7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d885bd8 sp=0xc00d885bb8 pc=0x4907d6 runtime.selectgo(0xc00d885dd0, 0xc00d885d78, 0xc0032ba150?, 0x0, 0x4000000000000000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00d885d18 sp=0xc00d885bd8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x624a948, 0xc0032ba150}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2247 +0x2c9 fp=0xc00d885f30 sp=0xc00d885d18 pc=0x1e332c9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00d885fe0 sp=0xc00d885f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d885fe8 sp=0xc00d885fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 961 [select]: runtime.gopark(0xc00b5b9dd0?, 0x2?, 0x48?, 0xa9?, 0xc00b5b9d7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b2abd8 sp=0xc001b2abb8 pc=0x4907d6 runtime.selectgo(0xc001b2add0, 0xc00b5b9d78, 0xc0032ba1b0?, 0x0, 0x4000000000000000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001b2ad18 sp=0xc001b2abd8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x624a948, 0xc0032ba1b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2247 +0x2c9 fp=0xc001b2af30 sp=0xc001b2ad18 pc=0x1e332c9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001b2afe0 sp=0xc001b2af30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b2afe8 sp=0xc001b2afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498413 [semacquire]: runtime.gopark(0xae1b735ec4f?, 0x8fc4c00?, 0x20?, 0xb2?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e587708 sp=0xc00e5876e8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x34?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00e587770 sp=0xc00e587708 pc=0x4a17de sync.runtime_SemacquireMutex(0xc00e587830?, 0xe0?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00e5877a0 sp=0xc00e587770 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x7f271e737420?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00e5877f8 sp=0xc00e5877a0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc0109fc000?, 0xc00e587888?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00e587840 sp=0xc00e5877f8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0109fc000, {0x6?, 0x6?}, 0x0, 0xc00f619178, 0xc004ddbc10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00e587940 sp=0xc00e587840 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc00f618f00, 0x40?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00e5879e0 sp=0xc00e587940 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc00f618f00, {0xc004363e40, 0x1e, 0x20}, 0xc00809a6a0?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc00e587a90 sp=0xc00e5879e0 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc00809a6a0?, 0x0?, 0x0?}, {0xc004363e40?, 0x1e?, 0xc00548bce0?}, 0x30?, 0xd3?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc00e587af8 sp=0xc00e587a90 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc004ddd1e0?, {0xc00809a6a0?, 0x1?, 0x20?}, {0xc004363e40?, 0x18?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00e587b48 sp=0xc00e587af8 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc004ddd1e0, {0xc00809a6a0, 0x1e, 0x20}, {0xc004363e40, 0x1e, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00e587b98 sp=0xc00e587b48 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc004ddbe20, {0xc004363e40?, 0xffffffffffffffe8?, 0x0?}, 0x0?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00e587cc8 sp=0xc00e587b98 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc004ddbe20, {0xc00809a6a0?, 0xc006d0c600?, 0xc455de?}, {0xc004363e40?, 0xc001aacb00?, 0xc0063a7448?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00e587d08 sp=0xc00e587cc8 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc004ddbc88, {0xc00809a6a0?, 0x1b?, 0x20?}, {0xc004363e40, 0x1e, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00e587d60 sp=0xc00e587d08 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc004ddb900, {0xc004363e40, 0x1e, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00e587df8 sp=0xc00e587d60 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0076f1608, {{0xc007fdbf20, 0x1d, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00e587e48 sp=0xc00e587df8 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00af44000, {0x624a948, 0xc040030840}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc00e587eb0 sp=0xc00e587e48 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc040030840}, {0x62b4fe0, 0xc0076f1608}, {0xc007fdbf20, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc00e588480 sp=0xc00e587eb0 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc040030840}, {0x628f760?, 0xc0076f1600?}, {0xc007fdbf20, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc00e588600 sp=0xc00e588480 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc040030840?}, {0x628f760?, 0xc0076f1600?}, {0xc007fdbf20?, 0xc00e5887a0?, 0x4618e7?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc00e5886f0 sp=0xc00e588600 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0?, {0x624a948, 0xc040030840}, {0x628f760, 0xc0076f1600}, {0xf0, 0xe, 0x3a, 0xb3, 0x57, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:95 +0xd3 fp=0xc00e5887b0 sp=0xc00e5886f0 pc=0x1c96133 github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0xf0, 0xe, 0x3a, 0xb3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:2034 +0xbc fp=0xc00e588998 sp=0xc00e5887b0 pc=0x1d8fb1c github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc010617680, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:206 +0x36f fp=0xc00e58a808 sp=0xc00e588998 pc=0x1dc43af github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00e58b278, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc00e58ae78 sp=0xc00e58a808 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc006f10000, {0x624a948, 0xc040030840}, 0xc010617680, 0xc012b15cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc00e58bb08 sp=0xc00e58ae78 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc040030840}, 0xc010617680, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00e58be28 sp=0xc00e58bb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76c3749d58, 0x0, 0x0}, 0x0, {0x17600a76e3dd0444, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00e58c1f0 sp=0xc00e58be28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76c3749d58, 0x0, 0x0}, 0x0, {0x17600a76e3dd0444, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00e58cb78 sp=0xc00e58c1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xff3d15f6d74568b4?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3dd0444, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00e58cdc0 sp=0xc00e58cb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0400307e0}, {0xc0016bd508?}, 0xc010617590) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00e58d430 sp=0xc00e58cdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc040030720}, 0xc010617590) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00e58d4d8 sp=0xc00e58d430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc040030720}, {0x52f0080?, 0xc010617590}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00e58d518 sp=0xc00e58d4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc040030720}, {0x52f0080, 0xc010617590}, 0xc003402ae0, 0xc004feb338) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00e58d848 sp=0xc00e58d518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc040030720?}, {0x52f0080?, 0xc010617590?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00e58d888 sp=0xc00e58d848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc040030720}, {0x52f0080, 0xc010617590}, 0xc00e58d920?, 0xc003b9de00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00e58d8b8 sp=0xc00e58d888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc040030720?}, {0x52f0080?, 0xc010617590?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e58d8f8 sp=0xc00e58d8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc040030720?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00e58d930 sp=0xc00e58d8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc040030720}, {0x203000?, 0x40?}, 0xc00e58d9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00e58d9a8 sp=0xc00e58d930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc040030720?}, {0x52f0080?, 0xc010617590?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00e58da20 sp=0xc00e58d9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc040030720?}, {0x52f0080?, 0xc010617590?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e58da60 sp=0xc00e58da20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc040030720}, {0x52f0080, 0xc010617590}, 0xc003402ae0, 0xc004feb338) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00e58dab0 sp=0xc00e58da60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc040030720}, 0xc011f07020, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00e58db08 sp=0xc00e58dab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc00cdebd40, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00e58de48 sp=0xc00e58db08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc00cdebd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00e58df68 sp=0xc00e58de48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00e58dfe0 sp=0xc00e58df68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e58dfe8 sp=0xc00e58dfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 963 [select, 6 minutes]: runtime.gopark(0xc00df4fee8?, 0x3?, 0x5c?, 0x0?, 0xc00df4fd7a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00df4fbe8 sp=0xc00df4fbc8 pc=0x4907d6 runtime.selectgo(0xc00df4fee8, 0xc00df4fd74, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00df4fd28 sp=0xc00df4fbe8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1({0x624a948, 0xc0032ba270}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2347 +0x2ce fp=0xc00df4ff30 sp=0xc00df4fd28 pc=0x1e33ace github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00df4ffe0 sp=0xc00df4ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00df4ffe8 sp=0xc00df4ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 964 [select]: runtime.gopark(0xc001579d50?, 0x3?, 0x69?, 0x0?, 0xc001579d12?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001579b70 sp=0xc001579b50 pc=0x4907d6 runtime.selectgo(0xc001579d50, 0xc001579d0c, 0x0?, 0x0, 0x7febcc7691?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001579cb0 sp=0xc001579b70 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1.2({0x624a8a0, 0xc006028640}, {0xc006028640?, 0xbebc200?, 0x0?}, 0xc002702180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2396 +0x19f fp=0xc001579d90 sp=0xc001579cb0 pc=0x1e345bf github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1({0x624a948?, 0xc0032ba2d0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2457 +0x2f5 fp=0xc001579f30 sp=0xc001579d90 pc=0x1e34115 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001579fe0 sp=0xc001579f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001579fe8 sp=0xc001579fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 965 [select]: runtime.gopark(0xc006167e08?, 0x2?, 0xc5?, 0x7b?, 0xc006167ddc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dd1bbf0 sp=0xc00dd1bbd0 pc=0x4907d6 runtime.selectgo(0xc00dd1be08, 0xc006167dd8, 0xc0032ba3c0?, 0x0, 0xc005af9000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dd1bd30 sp=0xc00dd1bbf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1({0x624a948, 0xc0032ba3c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:169 +0x1ad fp=0xc00dd1bf30 sp=0xc00dd1bd30 pc=0x1e4b04d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00dd1bfe0 sp=0xc00dd1bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dd1bfe8 sp=0xc00dd1bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 341844 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x37a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b0a6f58 sp=0xc00b0a6f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbd700?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00b0a6f90 sp=0xc00b0a6f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd618, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00b0a6fb0 sp=0xc00b0a6f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc000933180?, 0xc0036b0000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00b0a6fd8 sp=0xc00b0a6fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000933180, {0xc0036b0000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00b0a7058 sp=0xc00b0a6fd8 pc=0x535efa net.(*netFD).Read(0xc000933180, {0xc0036b0000?, 0x7b5a68?, 0xc000933180?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00b0a70a0 sp=0xc00b0a7058 pc=0x7a20c9 net.(*conn).Read(0xc01185e4c8, {0xc0036b0000?, 0x164290f?, 0xc01185e4c8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00b0a70e8 sp=0xc00b0a70a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00f066960, {0xc0036b0000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00b0a7130 sp=0xc00b0a70e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc035b88048, {0xc0036b0000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00b0a71d0 sp=0xc00b0a7130 pc=0x370f896 bufio.(*Reader).fill(0xc00c2d5890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00b0a7208 sp=0xc00b0a71d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00c2d5890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00b0a7228 sp=0xc00b0a7208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00c2d5e68?, {0x62295c8, 0xc00c2d5890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00b0a7260 sp=0xc00b0a7228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00c2d5800, 0xc00fc00b30, 0xc00b0a7442, 0xc00b0a7444, {0x623b718, 0xc0200b0f30}, 0xc0059fa000, 0xc00ceaa714, 0xc00b0a7440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00b0a7398 sp=0xc00b0a7260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00c2d5800, {0x624a948, 0xc0434b8990}, 0xc00fc00b20, 0xc0059fa000, 0xc0434b8630, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00b0a7768 sp=0xc00b0a7398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0434b8990}, {0x628a538, 0xc00f066960}, {{{0xc004435038, 0x4}}, 0x1, 0xc0434b8900, 0xc0434b8930, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00b0a78a8 sp=0xc00b0a7768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0434b8990}, {0x628a538?, 0xc00f066960}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00b0a7e20 sp=0xc00b0a78a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0434b8360?}, {0x628a538, 0xc00f066960}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00b0a7eb0 sp=0xc00b0a7e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0434b8360}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00b0a7f30 sp=0xc00b0a7eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b0a7fe0 sp=0xc00b0a7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b0a7fe8 sp=0xc00b0a7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 922 [select]: runtime.gopark(0xc0056c3ed0?, 0x4?, 0x85?, 0x0?, 0xc0056c3df8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0106cfc50 sp=0xc0106cfc30 pc=0x4907d6 runtime.selectgo(0xc0106cfed0, 0xc0056c3df0, 0xc005ec05c0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0106cfd90 sp=0xc0106cfc50 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2({0x624a948?, 0xc003424540?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:168 +0x316 fp=0xc0106cff30 sp=0xc0106cfd90 pc=0x3752156 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0106cffe0 sp=0xc0106cff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0106cffe8 sp=0xc0106cffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 967 [select, 9 minutes]: runtime.gopark(0xc0032daee8?, 0x3?, 0xc0?, 0xa3?, 0xc0032dae72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032dacc8 sp=0xc0032daca8 pc=0x4907d6 runtime.selectgo(0xc0032daee8, 0xc0032dae6c, 0xa?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032dae08 sp=0xc0032dacc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032ba810}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc0032daf30 sp=0xc0032dae08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0032dafe0 sp=0xc0032daf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032dafe8 sp=0xc0032dafe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 968 [select]: runtime.gopark(0xc00a971ee8?, 0x3?, 0x0?, 0x4?, 0xc00a971e72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00157bcc8 sp=0xc00157bca8 pc=0x4907d6 runtime.selectgo(0xc00157bee8, 0xc00a971e6c, 0xc003486ed0?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00157be08 sp=0xc00157bcc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032ba870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc00157bf30 sp=0xc00157be08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00157bfe0 sp=0xc00157bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00157bfe8 sp=0xc00157bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 969 [select]: runtime.gopark(0xc003a69ee8?, 0x3?, 0x0?, 0xd4?, 0xc003a69e72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005febcc8 sp=0xc005febca8 pc=0x4907d6 runtime.selectgo(0xc005febee8, 0xc003a69e6c, 0xc01b04e390?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005febe08 sp=0xc005febcc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032ba8d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc005febf30 sp=0xc005febe08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005febfe0 sp=0xc005febf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005febfe8 sp=0xc005febfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 970 [select]: runtime.gopark(0xc003f97ee8?, 0x3?, 0x0?, 0x4?, 0xc003f97e72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cb6fcc8 sp=0xc00cb6fca8 pc=0x4907d6 runtime.selectgo(0xc00cb6fee8, 0xc003f97e6c, 0xc01fd50b70?, 0x0, 0x2b?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00cb6fe08 sp=0xc00cb6fcc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032ba930}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc00cb6ff30 sp=0xc00cb6fe08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cb6ffe0 sp=0xc00cb6ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cb6ffe8 sp=0xc00cb6ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 971 [select]: runtime.gopark(0xc00462fee8?, 0x3?, 0x0?, 0x3c?, 0xc00462fe72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bbcdcc8 sp=0xc00bbcdca8 pc=0x4907d6 runtime.selectgo(0xc00bbcdee8, 0xc00462fe6c, 0xc0121f32c0?, 0x0, 0x2b?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bbcde08 sp=0xc00bbcdcc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032ba990}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc00bbcdf30 sp=0xc00bbcde08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00bbcdfe0 sp=0xc00bbcdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bbcdfe8 sp=0xc00bbcdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 972 [select]: runtime.gopark(0xc00a585ee8?, 0x3?, 0x0?, 0x54?, 0xc00a585e72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a585cc8 sp=0xc00a585ca8 pc=0x4907d6 runtime.selectgo(0xc00a585ee8, 0xc00a585e6c, 0xc012957dd0?, 0x0, 0x29?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a585e08 sp=0xc00a585cc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032ba9f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc00a585f30 sp=0xc00a585e08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a585fe0 sp=0xc00a585f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a585fe8 sp=0xc00a585fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 973 [select, 9 minutes]: runtime.gopark(0xc0032dcee8?, 0x3?, 0x1?, 0x0?, 0xc0032dce72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003630cc8 sp=0xc003630ca8 pc=0x4907d6 runtime.selectgo(0xc003630ee8, 0xc0032dce6c, 0x13?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003630e08 sp=0xc003630cc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032baa50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc003630f30 sp=0xc003630e08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003630fe0 sp=0xc003630f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003630fe8 sp=0xc003630fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 974 [select]: runtime.gopark(0xc002d93ee8?, 0x3?, 0x5?, 0x30?, 0xc002d93e72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d93cc8 sp=0xc002d93ca8 pc=0x4907d6 runtime.selectgo(0xc002d93ee8, 0xc002d93e6c, 0xc013bcdb90?, 0x0, 0x34?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d93e08 sp=0xc002d93cc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032baab0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc002d93f30 sp=0xc002d93e08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002d93fe0 sp=0xc002d93f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d93fe8 sp=0xc002d93fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 975 [select, 9 minutes]: runtime.gopark(0xc0032e9ee8?, 0x3?, 0x0?, 0x0?, 0xc0032e9e72?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003632cc8 sp=0xc003632ca8 pc=0x4907d6 runtime.selectgo(0xc003632ee8, 0xc0032e9e6c, 0x0?, 0x0, 0xaf163d8738086017?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003632e08 sp=0xc003632cc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x624a948, 0xc0032bab10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x118 fp=0xc003632f30 sp=0xc003632e08 pc=0x1d76158 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003632fe0 sp=0xc003632f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003632fe8 sp=0xc003632fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 976 [select]: runtime.gopark(0xc0079c9d30?, 0x3?, 0x5b?, 0x0?, 0xc0079c9cd2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079c9b08 sp=0xc0079c9ae8 pc=0x4907d6 runtime.selectgo(0xc0079c9d30, 0xc0079c9ccc, 0xc0032bab70?, 0x0, 0xc0018e1000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0079c9c48 sp=0xc0079c9b08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc006780460, {0x624a948, 0xc0032bab70}, {0x19b56a0?, 0x8fc5000?, 0x0?}, 0xc008393000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:224 +0x23c fp=0xc0079c9d70 sp=0xc0079c9c48 pc=0x1e2523c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc001491c00?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:286 +0x4a fp=0xc0079c9dc0 sp=0xc0079c9d70 pc=0x1e25d4a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc006795b30, 0xc0060285c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:452 +0x1e9 fp=0xc0079c9e50 sp=0xc0079c9dc0 pc=0x1e2b2e9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1({0x624a948?, 0xc0032bab70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:284 +0x23c fp=0xc0079c9f30 sp=0xc0079c9e50 pc=0x1e2599c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0079c9fe0 sp=0xc0079c9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079c9fe8 sp=0xc0079c9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 4832 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0xd?, 0x4?, 0xc014f9b640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc014f9b5d0 sp=0xc014f9b5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc0074ed490, 0xb) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc014f9b618 sp=0xc014f9b5d0 pc=0x4be30c sync.(*Cond).Wait(0xc0014d1400?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc014f9b650 sp=0xc014f9b618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0014d1a68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc014f9b708 sp=0xc014f9b650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0038d9900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc014f9bcb8 sp=0xc014f9b708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0038d9900, {0x624a8a0, 0xc0074ed540}, 0x411212b000000000?, 0xc008e0ddc0?, 0xc00090b160?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc014f9bd70 sp=0xc014f9bcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0074ed540?}, {0xc0059fa000?}, 0x3?, 0xc00431fde8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc014f9be20 sp=0xc014f9bd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc014f9bfe0 sp=0xc014f9be20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc014f9bfe8 sp=0xc014f9bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498133 [select]: runtime.gopark(0xc00cbade60?, 0x3?, 0x0?, 0x0?, 0xc00cbade32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cbadcb8 sp=0xc00cbadc98 pc=0x4907d6 runtime.selectgo(0xc00cbade60, 0xc00cbade2c, 0xc007ed3a40?, 0x0, 0xc005977098?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00cbaddf8 sp=0xc00cbadcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0084401d0, {0x624a948, 0xc007ed3a40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00cbadeb0 sp=0xc00cbaddf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc007ed3a40?}) :1 +0x39 fp=0xc00cbaded8 sp=0xc00cbadeb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc007ed3a40}, {0x53ddbcd?, 0x6294e00?}, 0xc0085cffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00cbadf50 sp=0xc00cbaded8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00cbadfe0 sp=0xc00cbadf50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cbadfe8 sp=0xc00cbadfe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 427180 [select]: runtime.gopark(0xc017d90128?, 0x2?, 0xf8?, 0xf6?, 0xc017d90124?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc017d8ffb0 sp=0xc017d8ff90 pc=0x4907d6 runtime.selectgo(0xc017d90128, 0xc017d90120, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc017d900f0 sp=0xc017d8ffb0 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc007e725a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc017d90158 sp=0xc017d900f0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ec98f20, {0x51b9a20?, 0xc011794280}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc017d90260 sp=0xc017d90158 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x100?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc017d90290 sp=0xc017d90260 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc007e72360, 0xc017d90368, 0xc017d90338) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc017d902e8 sp=0xc017d90290 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc007e72360, {0x51b9a20?, 0xc011794280?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc017d90398 sp=0xc017d902e8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc0083645a0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003dff770}, {0x51b9a20, 0xc011794280}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc017d90400 sp=0xc017d90398 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0083645a0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dff770}, {0x51b9a20, 0xc011794280}, 0x7f2e68c0ad34503d?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc017d904f0 sp=0xc017d90400 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x3ba78f70503b2957?, {0x624a948?, 0xc0083645a0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003dff770?}, {0x51b9a20?, 0xc011794280?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc017d90590 sp=0xc017d904f0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123bf080, {0x624a948, 0xc0083645a0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc017d90610 sp=0xc017d90590 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123bf080?}}, {0x624a948, 0xc0083645a0}, 0xc003dff770, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc017d906e8 sp=0xc017d90610 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x22666750?, {0x624a948?, 0xc0083645a0?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc022666848?}) :1 +0x65 fp=0xc017d90738 sp=0xc017d906e8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0113e98c0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc017d909d8 sp=0xc017d90738 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0113e98c0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e36b8f1b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc017d90b10 sp=0xc017d909d8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e36b8f1b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc017d91258 sp=0xc017d90b10 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e36b8f1b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc017d91b58 sp=0xc017d91258 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e36b8f1b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc017d923c0 sp=0xc017d91b58 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e36b8f1b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc017d927e0 sp=0xc017d923c0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc016881038, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc017d92a38 sp=0xc017d927e0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc016881000, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc017d92b60 sp=0xc017d92a38 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc016880fd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc017d92df0 sp=0xc017d92b60 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc016880ed0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc017d93390 sp=0xc017d92df0 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc016880ed0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc017d93680 sp=0xc017d93390 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc016880da0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc017d93980 sp=0xc017d93680 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc016880d80?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc017d93b28 sp=0xc017d93980 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc016880cd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc017d93d40 sp=0xc017d93b28 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc016880b00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc017d93f98 sp=0xc017d93d40 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc017d940c0 sp=0xc017d93f98 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0153080b0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc017d94388 sp=0xc017d940c0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc017d944a0 sp=0xc017d94388 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00bbf7800, {0x624a948, 0xc0083643c0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc017d946c0 sp=0xc017d944a0 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00bbf7800, {0x624a948, 0xc0083643c0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc017d949b8 sp=0xc017d946c0 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc015e01560, {0x624a948, 0xc0083643c0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc017d94a98 sp=0xc017d949b8 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).nextKey(0xc01377f8c0, {0x624a948?, 0xc0083643c0?}) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:665 +0x7b fp=0xc017d94c48 sp=0xc017d94a98 pc=0x232b69b github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).startScan(...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:635 github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).StartScan(0xc01377f8c0, {0x624a948, 0xc0083643c0}, {0xc008364510, 0x1?, 0x203008?}, {0xc0034aa140?, 0x11?, 0x45e42d?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:492 +0x18f fp=0xc017d94cc0 sp=0xc017d94c48 pc=0x232a82f github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc0083e6a00) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:999 +0xbca fp=0xc017d94ea0 sp=0xc017d94cc0 pc=0x258acca github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc0083e6a00) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:731 +0x70 fp=0xc017d94f20 sp=0xc017d94ea0 pc=0x25899d0 github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc00a038300, {{0x0?, 0x0?}, 0xc0083643f0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc017d94f98 sp=0xc017d94f20 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*errorIfRowsNode).Next(0xc011affd40, {{0x624a948?, 0xc008364360?}, 0xc011811000?, 0xc00a715878?}) github.com/cockroachdb/cockroach/pkg/sql/error_if_rows.go:42 +0x4b fp=0xc017d94fd0 sp=0xc017d94f98 pc=0x33e49cb github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc012086000) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc017d950c8 sp=0xc017d94fd0 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc00d3e4000) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:243 +0x13c fp=0xc017d95448 sp=0xc017d950c8 pc=0x299d6fc github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc00cdb8000) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc017d95460 sp=0xc017d95448 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc017d95478 sp=0xc017d95460 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc000f4ca80?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc017d954b8 sp=0xc017d95478 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc00cdb8000, {0x624a8a0?, 0xc005e4f480?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc017d95648 sp=0xc017d954b8 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc00b56d360, {0x624a8a0, 0xc005e4f480}, 0xc0153080b0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc017d956e8 sp=0xc017d95648 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc03f1f6090}, 0xc003dfeb40, 0xc0153080b0, 0xc011794000, 0xc008189880, 0xc00c104c00, 0xc00e2a8590) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc017d95f98 sp=0xc017d956e8 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).planAndRunPostquery(0xc0052d77c0, {0x624a948, 0xc03f1f6090}, {{0x624c6d0?, 0xc011affd40?}, 0x0?}, 0xc00a715878, 0xc00c104c00, 0x33b057f?, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:2044 +0x71e fp=0xc017d96148 sp=0xc017d95f98 pc=0x33bfdfe github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunCascadesAndChecks(0xc0052d77c0, {0x624a948, 0xc03f1f6090}, 0xc00a715878, 0xc012bedec0, 0xc00a715fd8, 0xc00aa74380) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1931 +0xe8c fp=0xc017d962f8 sp=0xc017d96148 pc=0x33bf44c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc00a715300?, {0x624a948, 0xc03f1f6090}, 0xc00a715bb0, 0xc00a715878?, 0xc00a715878, 0xc00aa74380, 0xc012bedec0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1488 +0x3aa fp=0xc017d96410 sp=0xc017d962f8 pc=0x33bc68a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00a715300, {0x624a948?, 0xc03f1f6090}, 0xc00a715878, 0xc03f1f6090?, {0x7f275f329680?, 0xc005ca9ec8}, 0xa7?, 0xc00a1eae58) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc017d96518 sp=0xc017d96410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00a715300, {0x624a8a0, 0xc012bed780}, 0xc00a715878, {0x7f275f329680, 0xc005ca9ec8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc017d96938 sp=0xc017d96518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00a715300, {0x624a948?, 0xc008de7d10?}, {{0x6273140, 0xc011a8b2c0}, {0xc0027795e9, 0x350}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc017d972c8 sp=0xc017d96938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc008de7d10?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc017d97380 sp=0xc017d972c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc008de7d10?}, {0x6273140?, 0xc011a8b2c0?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc017d974d0 sp=0xc017d97380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00a715300, {0x624a948, 0xc008de7d10}, {{0x6273140, 0xc011a8b2c0}, {0xc0027795e9, 0x350}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc017d97648 sp=0xc017d974d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6273140, 0xc011a8b2c0}, {0xc0027795e9, 0x350}, 0x0, 0x1}, {0xc1116e81942c4605, 0x7fea1b746e, 0x0}, {0xc1116e81942c55c0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc017d97708 sp=0xc017d97648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00a715300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc017d97cb8 sp=0xc017d97708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00a715300, {0x624a8a0, 0xc0009cb500}, 0x4118308000000000?, 0xc0064f1dc0?, 0xc00d4679c0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc017d97d70 sp=0xc017d97cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0009cb500?}, {0xc0059fa000?}, 0x3?, 0xc008b6a960?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc017d97e20 sp=0xc017d97d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc017d97fe0 sp=0xc017d97e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc017d97fe8 sp=0xc017d97fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 983 [runnable]: runtime.gopark(0xc0036bbd80?, 0xb?, 0x0?, 0x0?, 0x24b?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0036bba80 sp=0xc0036bba60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x3e?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0036bbab8 sp=0xc0036bba80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd438, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0036bbad8 sp=0xc0036bbab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00834b700?, 0xc003692000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0036bbb00 sp=0xc0036bbad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00834b700, {0xc003692000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0036bbb80 sp=0xc0036bbb00 pc=0x535efa net.(*netFD).Read(0xc00834b700, {0xc003692000?, 0x50c7cc0?, 0x489101?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0036bbbc8 sp=0xc0036bbb80 pc=0x7a20c9 net.(*conn).Read(0xc0061ec308, {0xc003692000?, 0x300?, 0xc00e89da10?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0036bbc10 sp=0xc0036bbbc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc007663c20, {0xc0079dd000, 0x9, 0xc0133f8ec4?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc0036bbc48 sp=0xc0036bbc10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc007663c20}, {0xc0079dd000, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc0036bbc90 sp=0xc0036bbc48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc0079dd000?, 0x9?, 0xc00e89d980?}, {0x62052c0?, 0xc007663c20?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc0036bbce0 sp=0xc0036bbc90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0079dcfc0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc0036bbd90 sp=0xc0036bbce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0032e2000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc0036bbfc8 sp=0xc0036bbd90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc0036bbfe0 sp=0xc0036bbfc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0036bbfe8 sp=0xc0036bbfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 984 [select]: runtime.gopark(0xc0017ebf00?, 0x2?, 0x56?, 0x0?, 0xc0017ebed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0017ebd60 sp=0xc0017ebd40 pc=0x4907d6 runtime.selectgo(0xc0017ebf00, 0xc0017ebed0, 0x7b55a5?, 0x0, 0xc003a4c000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0017ebea0 sp=0xc0017ebd60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bea910, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc0017ebf30 sp=0xc0017ebea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc007663c80) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc0017ebf88 sp=0xc0017ebf30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc0017ebfe0 sp=0xc0017ebf88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0017ebfe8 sp=0xc0017ebfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 924 [select]: runtime.gopark(0xc00ae49ef8?, 0x2?, 0x0?, 0xf0?, 0xc00ae49e4c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003631cb0 sp=0xc003631c90 pc=0x4907d6 runtime.selectgo(0xc003631ef8, 0xc00ae49e48, 0xc003424c30?, 0x0, 0x56844b0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003631df0 sp=0xc003631cb0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1({0x624a948, 0xc003424c30}) github.com/cockroachdb/cockroach/pkg/server/env_sampler.go:146 +0x1c5 fp=0xc003631f30 sp=0xc003631df0 pc=0x39a33a5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003631fe0 sp=0xc003631f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003631fe8 sp=0xc003631fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 925 [select, 1 minutes]: runtime.gopark(0xc00608eeb0?, 0x4?, 0x57?, 0x0?, 0xc00608edd8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00608ec20 sp=0xc00608ec00 pc=0x4907d6 runtime.selectgo(0xc00608eeb0, 0xc00608edd0, 0x0?, 0x0, 0xc0032c29c0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00608ed60 sp=0xc00608ec20 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc002941ce0, {0x624a948, 0xc003424c90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:175 +0x246 fp=0xc00608ef08 sp=0xc00608ed60 pc=0x3740cc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache-fm({0x624a948?, 0xc003424c90?}) :1 +0x39 fp=0xc00608ef30 sp=0xc00608ef08 pc=0x3741e99 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00608efe0 sp=0xc00608ef30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00608efe8 sp=0xc00608efe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 926 [select]: runtime.gopark(0xc0032dcfb0?, 0x2?, 0xae?, 0x3c?, 0xc0032dcf94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032dce20 sp=0xc0032dce00 pc=0x4907d6 runtime.selectgo(0xc0032dcfb0, 0xc0032dcf90, 0x1054e20?, 0x0, 0x21?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032dcf60 sp=0xc0032dce20 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc0032dcfe0 sp=0xc0032dcf60 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032dcfe8 sp=0xc0032dcfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 927 [select]: runtime.gopark(0xc00bffefb0?, 0x2?, 0x80?, 0xff?, 0xc00bffef84?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bffedf0 sp=0xc00bffedd0 pc=0x4907d6 runtime.selectgo(0xc00bffefb0, 0xc00bffef80, 0x0?, 0x0, 0x7fe860a3e3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bffef30 sp=0xc00bffedf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc00bffefe0 sp=0xc00bffef30 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bffefe8 sp=0xc00bffefe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 928 [select]: runtime.gopark(0xc0032e0fb0?, 0x2?, 0x1?, 0x30?, 0xc0032e0f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032e0e20 sp=0xc0032e0e00 pc=0x4907d6 runtime.selectgo(0xc0032e0fb0, 0xc0032e0f90, 0x1054e20?, 0x0, 0x6779a801?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032e0f60 sp=0xc0032e0e20 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x86 fp=0xc0032e0fe0 sp=0xc0032e0f60 pc=0x1696e66 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032e0fe8 sp=0xc0032e0fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x46a goroutine 929 [select]: runtime.gopark(0xc0032da7b0?, 0x2?, 0x6?, 0x0?, 0xc0032da784?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032da5f0 sp=0xc0032da5d0 pc=0x4907d6 runtime.selectgo(0xc0032da7b0, 0xc0032da780, 0x0?, 0x0, 0x7fe8607d97?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032da730 sp=0xc0032da5f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:424 +0x1d6 fp=0xc0032da7e0 sp=0xc0032da730 pc=0x1697316 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032da7e8 sp=0xc0032da7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:397 +0x56 goroutine 1010 [select]: runtime.gopark(0xc003636f78?, 0x2?, 0x0?, 0x0?, 0xc003636f0c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003636d48 sp=0xc003636d28 pc=0x4907d6 runtime.selectgo(0xc003636f78, 0xc003636f08, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003636e88 sp=0xc003636d48 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1() github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:102 +0xed fp=0xc003636fe0 sp=0xc003636e88 pc=0x168a14d runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003636fe8 sp=0xc003636fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:97 +0x1fe goroutine 1011 [select]: runtime.gopark(0xc0115244a8?, 0x2?, 0xc0?, 0xb5?, 0xc0115244a4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011524330 sp=0xc011524310 pc=0x4907d6 runtime.selectgo(0xc0115244a8, 0xc0115244a0, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc011524470 sp=0xc011524330 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc010ad8ea0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0115244d8 sp=0xc011524470 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0037b2420, {0x51b9a20?, 0xc007f9fb00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0115245e0 sp=0xc0115244d8 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x131?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc011524610 sp=0xc0115245e0 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc010ad8900, 0xc0115246e8, 0xc0115246b8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc011524668 sp=0xc011524610 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc010ad8900, {0x51b9a20?, 0xc007f9fb00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc011524718 sp=0xc011524668 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc000a83b60?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0106164b0}, {0x51b9a20, 0xc007f9fb00}, 0x0?, {0xc005bc7580, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc011524780 sp=0xc011524718 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc000a83b60}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0106164b0}, {0x51b9a20, 0xc007f9fb00}, 0x6141fef62e668027?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc011524870 sp=0xc011524780 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x3585af49a8661970?, {0x624a948?, 0xc000a83b60?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0106164b0?}, {0x51b9a20?, 0xc007f9fb00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc011524910 sp=0xc011524870 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0467621c0, {0x624a948, 0xc000a83b60}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc011524990 sp=0xc011524910 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0467621c0?}}, {0x624a948, 0xc000a83b60}, 0xc0106164b0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc011524a68 sp=0xc011524990 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x4848ae0?, {0x624a948?, 0xc000a83b60?}, 0x624a948?, {0x0?, 0xc005845e60?, 0xc005116bc8?}) :1 +0x65 fp=0xc011524ab8 sp=0xc011524a68 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0125d1680, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc011524d58 sp=0xc011524ab8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0125d1680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2e40a6a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc011524e90 sp=0xc011524d58 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2e40a6a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0115255d8 sp=0xc011524e90 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2e40a6a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc011525ed8 sp=0xc0115255d8 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2e40a6a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc011526740 sp=0xc011525ed8 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2e40a6a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc011526b60 sp=0xc011526740 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc015bdc638, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc011526db8 sp=0xc011526b60 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc015bdc600, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc011526ee0 sp=0xc011526db8 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc015bdc5d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x36e fp=0xc011527170 sp=0xc011526ee0 pc=0x19e4c4e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc015bdc4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc011527710 sp=0xc011527170 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc015bdc4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc011527a00 sp=0xc011527710 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc015bdc3a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc011527d00 sp=0xc011527a00 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc015bdc380?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc011527ea8 sp=0xc011527d00 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc015bdc2d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc0115280c0 sp=0xc011527ea8 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc015bdc100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc011528318 sp=0xc0115280c0 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc011528440 sp=0xc011528318 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0037b2210, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc011528708 sp=0xc011528440 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc011528818 sp=0xc011528708 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc018c53b90}, 0x0?, 0xc015bdd700) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc011528a10 sp=0xc011528818 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0037b2210, {0x624a948, 0xc018c53b90}, 0xc05354b458?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc011528a58 sp=0xc011528a10 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).updateLivenessAttempt.func1({0x624a948, 0xc018c53b90}, 0x62924c0?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:1379 +0x405 fp=0xc011528b60 sp=0xc011528a58 pc=0x1c47065 github.com/cockroachdb/cockroach/pkg/kv.runTxn.func1({0x624a948?, 0xc018c53b90?}, 0xc015bdc4d0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:980 +0x27 fp=0xc011528b88 sp=0xc011528b60 pc=0x16ae427 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0xc0037b2210, {0x624a948, 0xc018c53b90}, 0xc011528ec8) github.com/cockroachdb/cockroach/pkg/kv/txn.go:961 +0xae fp=0xc011528e70 sp=0xc011528b88 pc=0x16b6d0e github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x624a948, 0xc018c53b90}, 0x7?, 0x14d7f3c?) github.com/cockroachdb/cockroach/pkg/kv/db.go:979 +0x5a fp=0xc011528ee8 sp=0xc011528e70 pc=0x16ae33a github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x4?, {0x624a948, 0xc018c53b90}, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:942 +0x89 fp=0xc011528f30 sp=0xc011528ee8 pc=0x16ae0c9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:921 github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).updateLivenessAttempt(0xc00a1b74a0, {0x624a948, 0xc018c53b90}, {{0x4, 0x1b, {0x17600a78faa59581, 0x0, 0x0}, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:1352 +0x2da fp=0xc011529150 sp=0xc011528f30 pc=0x1c4659a github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).updateLiveness(0xc00a1b74a0, {0x624a948, 0xc018c53b90}, {{0x4, 0x1b, {0x17600a78faa59581, 0x0, 0x0}, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:1307 +0x338 fp=0xc011529498 sp=0xc011529150 pc=0x1c45bd8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).heartbeatInternal(0xc00a1b74a0, {0x624a910?, 0xc00d1deae0?}, {0x4, 0x1b, {0x17600a77f00f1889, 0x0, 0x0}, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:963 +0x885 fp=0xc011529a28 sp=0xc011529498 pc=0x1c42c65 github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1.1({0x624a910, 0xc00d1deae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:778 +0x46b fp=0xc011529ce8 sp=0xc011529a28 pc=0x1c41c4b github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x624a948?, 0xc003425230?}, {0x539c3ad, 0x17}, 0x10c388d00, 0xc00f54fec0) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed fp=0xc011529d88 sp=0xc011529ce8 pc=0x16210cd github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1({0xc0019fc000, 0xc0032ba9f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:761 +0x2a8 fp=0xc011529f30 sp=0xc011529d88 pc=0x1c41628 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011529fe0 sp=0xc011529f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011529fe8 sp=0xc011529fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 344683 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc0184bd640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0201ed5d0 sp=0xc0201ed5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc0081edf50, 0x458) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0201ed618 sp=0xc0201ed5d0 pc=0x4be30c sync.(*Cond).Wait(0xc003492800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0201ed650 sp=0xc0201ed618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc003492e68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc0201ed708 sp=0xc0201ed650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003306000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc0201edcb8 sp=0xc0201ed708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003306000, {0x624a8a0, 0xc0081edf80}, 0x4119b21000000000?, 0xc0079cbdc0?, 0xc008e75560?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0201edd70 sp=0xc0201edcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0081edf80?}, {0xc0059fa000?}, 0x3?, 0xc0008ac078?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0201ede20 sp=0xc0201edd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0201edfe0 sp=0xc0201ede20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0201edfe8 sp=0xc0201edfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 120471 [runnable]: runtime.gopark(0x0?, 0xb?, 0x0?, 0x0?, 0x281?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003eb0aa0 sp=0xc003eb0a80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0xbef6f1?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc003eb0ad8 sp=0xc003eb0aa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd528, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc003eb0af8 sp=0xc003eb0ad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc017192100?, 0xc001486000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc003eb0b20 sp=0xc003eb0af8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc017192100, {0xc001486000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc003eb0ba0 sp=0xc003eb0b20 pc=0x535efa net.(*netFD).Read(0xc017192100, {0xc001486000?, 0xc003eb0c70?, 0x46159f?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc003eb0be8 sp=0xc003eb0ba0 pc=0x7a20c9 net.(*conn).Read(0xc00de7c2b0, {0xc001486000?, 0x7f278cfaa108?, 0x18?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc003eb0c30 sp=0xc003eb0be8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00d119440, {0xc001486000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc003eb0c78 sp=0xc003eb0c30 pc=0x1641f96 bufio.(*Reader).Read(0xc00d1194a0, {0xc004692820, 0x9, 0xc004fea4f8?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc003eb0cb0 sp=0xc003eb0c78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc00d1194a0}, {0xc004692820, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc003eb0cf8 sp=0xc003eb0cb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004692820?, 0x9?, 0xc000198110?}, {0x62052c0?, 0xc00d1194a0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc003eb0d48 sp=0xc003eb0cf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0046927e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc003eb0df8 sp=0xc003eb0d48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00c483040, 0xc00395bfd0?, 0x1054c46?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc003eb0f18 sp=0xc003eb0df8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc00c483040}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc003eb0f80 sp=0xc003eb0f18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc003eb0fe0 sp=0xc003eb0f80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003eb0fe8 sp=0xc003eb0fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 498464 [select]: runtime.gopark(0xc00fd332e8?, 0x3?, 0x4?, 0x0?, 0xc00fd32fb2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fd32d38 sp=0xc00fd32d18 pc=0x4907d6 runtime.selectgo(0xc00fd332e8, 0xc00fd32fac, 0x17600a76c71a6334?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fd32e78 sp=0xc00fd32d38 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc03064b650}, 0xc0120d2870, 0xc00d1b57c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00fd335e8 sp=0xc00fd32e78 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc03064b650}, 0xc0120d2870, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00fd33908 sp=0xc00fd335e8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76c71a6334, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00fd33cd0 sp=0xc00fd33908 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76c71a6334, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00fd34658 sp=0xc00fd33cd0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00fd348a0 sp=0xc00fd34658 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc03064b5f0}, {0x4b29600?}, 0xc0120d2780) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00fd34f10 sp=0xc00fd348a0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc03064b560}, 0xc0120d2780) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00fd34fb8 sp=0xc00fd34f10 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc03064b560?}, {0x52f0080?, 0xc0120d2780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00fd34fe8 sp=0xc00fd34fb8 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc03064b560}, {0x52f0080, 0xc0120d2780}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00fd35318 sp=0xc00fd34fe8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03064b560?}, {0x52f0080?, 0xc0120d2780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00fd35358 sp=0xc00fd35318 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc03064b560}, {0x52f0080, 0xc0120d2780}, 0xc00ba833f0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00fd35388 sp=0xc00fd35358 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03064b560?}, {0x52f0080?, 0xc0120d2780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00fd353c8 sp=0xc00fd35388 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc03064b560?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00fd35400 sp=0xc00fd353c8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc03064b560}, {0xc00601a000?, 0x53d9bbc?}, 0xc00ba834b8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00fd35478 sp=0xc00fd35400 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc03064b560?}, {0x52f0080?, 0xc0120d2780?}, 0x2?, 0x52f0080?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00fd354f0 sp=0xc00fd35478 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03064b560?}, {0x52f0080?, 0xc0120d2780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00fd35530 sp=0xc00fd354f0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc03064b560?}, {0xc03064b560?, 0x4?}, {0x52f0080?, 0xc0120d2780?}, {0x51b9a20?, 0xc00d502900}, 0x20300c?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00fd35570 sp=0xc00fd35530 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03064b560}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0120d2780}, {0x51b9a20, 0xc00d502900}, 0xc00ba836e8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00fd35660 sp=0xc00fd35570 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc03064b560}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0120d2780}, {0x51b9a20, 0xc00d502900}, 0xc00ba837c8?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00fd356f8 sp=0xc00fd35660 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc03064b4a0}, 0xc0120d2780, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00fd35778 sp=0xc00fd356f8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x0?, {0x624a948?, 0xc03064b4a0?}, 0xba83830?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00fd35800 sp=0xc00fd35778 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc013c07440, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00fd35aa0 sp=0xc00fd35800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc013c07440, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00fd35bd8 sp=0xc00fd35aa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00fd36320 sp=0xc00fd35bd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00fd36c20 sp=0xc00fd36320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00fd37488 sp=0xc00fd36c20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3d5b043, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00fd378a8 sp=0xc00fd37488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00ab46538, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00fd37b00 sp=0xc00fd378a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00ab46500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00fd37c28 sp=0xc00fd37b00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc03064b260}, {0x7f275f30db68, 0xc00ab46500}, 0xc01957a120, {0xc01200d800, 0x14, 0x15}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc00fd37e60 sp=0xc00fd37c28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc03064b260}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc00fd37f30 sp=0xc00fd37e60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00fd37fe0 sp=0xc00fd37f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fd37fe8 sp=0xc00fd37fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 399941 [sleep]: runtime.gopark(0x4b8b5bcd9fd?, 0x2057de?, 0xd6?, 0xa5?, 0x8090483236?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0056c3df0 sp=0xc0056c3dd0 pc=0x4907d6 time.Sleep(0x1369dc137) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc0056c3e30 sp=0xc0056c3df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc001724360, 0x2057de, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc0056c3e88 sp=0xc0056c3e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc001724360, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc0056c3ec8 sp=0xc0056c3e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00f4db500?, 0x3a, 0xc003636fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc0056c3fa8 sp=0xc0056c3ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc0056c3fe0 sp=0xc0056c3fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0056c3fe8 sp=0xc0056c3fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 1066 [select]: runtime.gopark(0xc00a447ee8?, 0x3?, 0x61?, 0x0?, 0xc00a447ea2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00608ad28 sp=0xc00608ad08 pc=0x4907d6 runtime.selectgo(0xc00608aee8, 0xc00a447e9c, 0xc003d67a40?, 0x0, 0x6295c98?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00608ae68 sp=0xc00608ad28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber.(*KVSubscriber).Start.func1({0x624a948, 0xc003d67a40}) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go:271 +0x22f fp=0xc00608af30 sp=0xc00608ae68 pc=0x377340f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00608afe0 sp=0xc00608af30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00608afe8 sp=0xc00608afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498484 [select]: runtime.gopark(0xc008a4dc18?, 0x3?, 0xd2?, 0xd9?, 0xc008a4db3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008a4d8f0 sp=0xc008a4d8d0 pc=0x4907d6 runtime.selectgo(0xc008a4dc18, 0xc008a4db34, 0xc00511d440?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008a4da30 sp=0xc008a4d8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc00564c7b0}, {0x6221560, 0xc026467200}, {0x46?, {0xc008a4de01?, 0x8a4de60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc008a4dda0 sp=0xc008a4da30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc00564c7b0?}, 0x164, 0xc026467200, 0x1a?, 0xc01283e150) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc008a4de70 sp=0xc008a4dda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc00564c7b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc008a4df30 sp=0xc008a4de70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc008a4dfe0 sp=0xc008a4df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008a4dfe8 sp=0xc008a4dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1015 [runnable]: runtime.gopark(0xc00df33950?, 0x4?, 0xee?, 0x1?, 0xc00df338a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00df33700 sp=0xc00df336e0 pc=0x4907d6 runtime.selectgo(0xc00df33950, 0xc00df33898, 0xc03bfda1a0?, 0x0, 0x4?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00df33840 sp=0xc00df33700 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000c62700, 0xc00605d290, {0x628a900, 0xc005ec6310}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:434 +0x345 fp=0xc00df33cd8 sp=0xc00df33840 pc=0x1d83c25 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x624a948, 0xc003c92d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:601 +0x367 fp=0xc00df33e18 sp=0xc00df33cd8 pc=0x1d852e7 runtime/pprof.Do({0x624a948?, 0xc003c925d0?}, {{0xc004419740?, 0xc000097eb8?, 0xc448e6?}}, 0xc0044191a0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00df33e88 sp=0xc00df33e18 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x624a948, 0xc003c925d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:607 +0x1c8 fp=0xc00df33f30 sp=0xc00df33e88 pc=0x1d84f28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00df33fe0 sp=0xc00df33f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00df33fe8 sp=0xc00df33fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 117855 [select]: runtime.gopark(0xc00e78fdb8?, 0x3?, 0x70?, 0x0?, 0xc00e78fd32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e78fb90 sp=0xc00e78fb70 pc=0x4907d6 runtime.selectgo(0xc00e78fdb8, 0xc00e78fd2c, 0xc00f5537a0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00e78fcd0 sp=0xc00e78fb90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc00f5537a0}, 0xc00f554690, {0xc005e769f0, 0x12}, 0xc003bf74a0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc00e78fe68 sp=0xc00e78fcd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc00f5537a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc00e78ff30 sp=0xc00e78fe68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00e78ffe0 sp=0xc00e78ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e78ffe8 sp=0xc00e78ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115787 [select, 6 minutes]: runtime.gopark(0xc00608dfb0?, 0x2?, 0x0?, 0x0?, 0xc00608df9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00608de28 sp=0xc00608de08 pc=0x4907d6 runtime.selectgo(0xc00608dfb0, 0xc00608df98, 0x1000000010000?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00608df68 sp=0xc00608de28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc00608dfe0 sp=0xc00608df68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00608dfe8 sp=0xc00608dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 115786 [select, 6 minutes]: runtime.gopark(0xc004d39fb0?, 0x2?, 0x0?, 0x0?, 0xc004d39f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004d39e18 sp=0xc004d39df8 pc=0x4907d6 runtime.selectgo(0xc004d39fb0, 0xc004d39f90, 0xc004d39fd0?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004d39f58 sp=0xc004d39e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc004d39fe0 sp=0xc004d39f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004d39fe8 sp=0xc004d39fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 425810 [sleep]: runtime.gopark(0x4b8d0037b6d?, 0x400ea1?, 0x38?, 0xbd?, 0x80c5811395?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a6afdf0 sp=0xc00a6afdd0 pc=0x4907d6 time.Sleep(0x151900392) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a6afe30 sp=0xc00a6afdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0043a6f30, 0x400ea1, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a6afe88 sp=0xc00a6afe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0043a6f30, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a6afec8 sp=0xc00a6afe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc0019de500?, 0x1d, 0xc005241f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a6affa8 sp=0xc00a6afec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a6affe0 sp=0xc00a6affa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a6affe8 sp=0xc00a6affe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 492846 [select]: runtime.gopark(0xc01569c768?, 0x2?, 0x90?, 0xef?, 0xc01569c764?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01569c5f0 sp=0xc01569c5d0 pc=0x4907d6 runtime.selectgo(0xc01569c768, 0xc01569c760, 0x10?, 0x0, 0x301?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01569c730 sp=0xc01569c5f0 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc04a4db320) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01569c798 sp=0xc01569c730 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ec988f0, {0x51b9a20?, 0xc010b7bb80}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01569c8a0 sp=0xc01569c798 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x36c?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01569c8d0 sp=0xc01569c8a0 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc04a4dac60, 0xc01569c9a8, 0xc01569c978) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01569c928 sp=0xc01569c8d0 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc04a4dac60, {0x51b9a20?, 0xc010b7bb80?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01569c9d8 sp=0xc01569c928 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc0161714d0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003dfe000}, {0x51b9a20, 0xc010b7bb80}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01569ca40 sp=0xc01569c9d8 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0161714d0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dfe000}, {0x51b9a20, 0xc010b7bb80}, 0x20da58f034226dd7?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01569cb30 sp=0xc01569ca40 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x14205a629ad164e5?, {0x624a948?, 0xc0161714d0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003dfe000?}, {0x51b9a20?, 0xc010b7bb80?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01569cbd0 sp=0xc01569cb30 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123beec8, {0x624a948, 0xc0161714d0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01569cc50 sp=0xc01569cbd0 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123beec8?}}, {0x624a948, 0xc0161714d0}, 0xc003dfe000, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01569cd28 sp=0xc01569cc50 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x1569cd90?, {0x624a948?, 0xc0161714d0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc01569ce88?}) :1 +0x65 fp=0xc01569cd78 sp=0xc01569cd28 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc010df5b60, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01569d018 sp=0xc01569cd78 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc010df5b60, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3592e99, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01569d150 sp=0xc01569d018 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3592e99, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01569d898 sp=0xc01569d150 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3592e99, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01569e198 sp=0xc01569d898 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3592e99, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc01569ea00 sp=0xc01569e198 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3592e99, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01569ee20 sp=0xc01569ea00 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00f601b38, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc01569f078 sp=0xc01569ee20 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00f601b00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc01569f1a0 sp=0xc01569f078 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00f601ad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc01569f430 sp=0xc01569f1a0 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00f6019d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc01569f9d0 sp=0xc01569f430 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00f6019d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc01569fcc0 sp=0xc01569f9d0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00f6018a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc01569ffc0 sp=0xc01569fcc0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00f601880?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc0156a0168 sp=0xc01569ffc0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00f6017d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc0156a0380 sp=0xc0156a0168 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00f601600, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc0156a05d8 sp=0xc0156a0380 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc0156a0700 sp=0xc0156a05d8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0050cfad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc0156a09c8 sp=0xc0156a0700 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc0156a0ad8 sp=0xc0156a09c8 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc018c525d0}, 0x8?, 0xc00d314000) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc0156a0cd0 sp=0xc0156a0ad8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0050cfad0, {0x624a948, 0xc018c525d0}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc0156a0d18 sp=0xc0156a0cd0 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00ac71c28, {0x624a948, 0xc018c525d0}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc0156a0d50 sp=0xc0156a0d18 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00ac71c00, {{0x624a948, 0xc018c525d0}, 0xc005ef7800, 0xc00c4ed178}) github.com/cockroachdb/cockroach/pkg/sql/update.go:197 +0x225 fp=0xc0156a0db0 sp=0xc0156a0d50 pc=0x352d9a5 github.com/cockroachdb/cockroach/pkg/sql.(*serializeNode).Next(0xc0152cfcb0, {{0x624a948?, 0xc018c525d0?}, 0xc005ef7800?, 0xc00c4ed178?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:115 +0x5a fp=0xc0156a0de8 sp=0xc0156a0db0 pc=0x3476ffa github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc00172d080) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc0156a0ee0 sp=0xc0156a0de8 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc00d3e5800) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:243 +0x13c fp=0xc0156a1260 sp=0xc0156a0ee0 pc=0x299d6fc github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc008a8a9a0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:202 +0x2e fp=0xc0156a12b0 sp=0xc0156a1260 pc=0x2860bee github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:48 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc017727860) github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:195 +0x4a fp=0xc0156a1410 sp=0xc0156a12b0 pc=0x299be6a github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc00d1de3c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f fp=0xc0156a14d0 sp=0xc0156a1410 pc=0x293bf5f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc00e3680b0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc0156a14e8 sp=0xc0156a14d0 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc0156a1500 sp=0xc0156a14e8 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc010ce0380?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc0156a1540 sp=0xc0156a1500 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc00e3680b0, {0x624a8a0?, 0xc012525b00?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc0156a16d0 sp=0xc0156a1540 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc00ec525a0, {0x624a8a0, 0xc012525b00}, 0xc0050cfad0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc0156a1770 sp=0xc0156a16d0 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc0152cfdd0}, 0xc0107d1950, 0xc0050cfad0, 0xc013662280, 0xc00d5b9500, 0xc00c4ed4b0, 0xc015332040) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc0156a2020 sp=0xc0156a1770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc0152cfdd0}, 0xe8?, 0xc0107d1950, 0x46bb4e3d77fe3701?, {{0x624ce40?, 0xc005281c20?}, 0x0?}, 0xc00d5b9500, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0156a20e0 sp=0xc0156a2020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc0152cfdd0}, 0xc00c4ed4b0, 0xc00c4ed178?, 0xc00c4ed178, 0xc00d5b9500, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc0156a21f8 sp=0xc0156a20e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00c4ecc00, {0x624a948?, 0xc0152cfdd0}, 0xc00c4ed178, 0xc0152cfdd0?, {0x7f275f329680?, 0xc00e8d7ec8}, 0x270?, 0xc0113f3be8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc0156a2300 sp=0xc0156a21f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00c4ecc00, {0x624a8a0, 0xc012524880}, 0xc00c4ed178, {0x7f275f329680, 0xc00e8d7ec8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc0156a2720 sp=0xc0156a2300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00c4ecc00, {0x624a948?, 0xc013a023c0?}, {{0x6274800, 0xc00e341980}, {0xc002ca1971, 0x270}, 0x6, 0x1}, 0xc034afb9e0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0156a30b0 sp=0xc0156a2720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc013a023c0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc0156a3168 sp=0xc0156a30b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc013a023c0?}, {0x6274800?, 0xc00e341980?}, 0x624a948?, 0xc013a023c0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0156a32b8 sp=0xc0156a3168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00c4ecc00, {0x624a948, 0xc013a023c0}, {{0x6274800, 0xc00e341980}, {0xc002ca1971, 0x270}, 0x6, 0x1}, 0xc034afb9e0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc0156a3430 sp=0xc0156a32b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00c4ecc00, {0x624a948, 0xc013a023c0}, {0xc034afb9e0, {0xc013667f20, 0x6, 0x6}, {0xc00809aa20, 0xf, 0xf}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc0156a3550 sp=0xc0156a3430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc002ca1ddc, 0x0}, 0x0, {0xc1116e8194ca9354, 0x7feab9c1a7, 0x0}, 0x1}, 0xc00c4ecc00, 0xc0156a38c0, 0xc0156a38b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc0156a3708 sp=0xc0156a3550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00c4ecc00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc0156a3cb8 sp=0xc0156a3708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00c4ecc00, {0x624a8a0, 0xc008037fc0}, 0x412ecc8200000000?, 0xc004a8fdc0?, 0xc00c8ded30?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0156a3d70 sp=0xc0156a3cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc008037fc0?}, {0xc0059fa000?}, 0x3?, 0xc0054ce1e0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0156a3e20 sp=0xc0156a3d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0156a3fe0 sp=0xc0156a3e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0156a3fe8 sp=0xc0156a3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 1020 [select, 9 minutes]: runtime.gopark(0xc0036b6f98?, 0x2?, 0x0?, 0x0?, 0xc0036b6f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0036b6e00 sp=0xc0036b6de0 pc=0x4907d6 runtime.selectgo(0xc0036b6f98, 0xc0036b6f78, 0xc00605dc60?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0036b6f40 sp=0xc0036b6e00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc003ca8040) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc0036b6fc8 sp=0xc0036b6f40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc0036b6fe0 sp=0xc0036b6fc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0036b6fe8 sp=0xc0036b6fe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 427544 [sleep]: runtime.gopark(0x4b85f70fd4d?, 0x200916?, 0x49?, 0x16?, 0x80279e0ea9?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aa1df0 sp=0xc004aa1dd0 pc=0x4907d6 time.Sleep(0x1243f7992) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004aa1e30 sp=0xc004aa1df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00c9833e0, 0x200916, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004aa1e88 sp=0xc004aa1e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00c9833e0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004aa1ec8 sp=0xc004aa1e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00c4e8000?, {0xc00c555500?, 0x27, 0xc00c982630?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004aa1fa8 sp=0xc004aa1ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004aa1fe0 sp=0xc004aa1fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aa1fe8 sp=0xc004aa1fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 1022 [select]: runtime.gopark(0xc00a5c3db8?, 0x3?, 0x66?, 0x0?, 0xc00a5c3d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a5c3b90 sp=0xc00a5c3b70 pc=0x4907d6 runtime.selectgo(0xc00a5c3db8, 0xc00a5c3d2c, 0xc003c930e0?, 0x0, 0x203001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a5c3cd0 sp=0xc00a5c3b90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc003c930e0}, 0xc006a4a0f0, {0xc005e768e8, 0x12}, 0xc0027744e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc00a5c3e68 sp=0xc00a5c3cd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc003c930e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc00a5c3f30 sp=0xc00a5c3e68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a5c3fe0 sp=0xc00a5c3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a5c3fe8 sp=0xc00a5c3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 465857 [sleep]: runtime.gopark(0x4b8503be73c?, 0x20ab6?, 0x17?, 0xc9?, 0x8010a2f76a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00697edf0 sp=0xc00697edd0 pc=0x4907d6 time.Sleep(0x11c7979cf) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00697ee30 sp=0xc00697edf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc010e11fb0, 0x20ab6, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00697ee88 sp=0xc00697ee30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc010e11fb0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00697eec8 sp=0xc00697ee88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00697ef80?, {0xc00342e000?, 0x32, 0xc00697ef70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00697efa8 sp=0xc00697eec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00697efe0 sp=0xc00697efa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00697efe8 sp=0xc00697efe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 1044 [select]: runtime.gopark(0xc000097f48?, 0x2?, 0x6?, 0x0?, 0xc000097ef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000097d50 sp=0xc000097d30 pc=0x4907d6 runtime.selectgo(0xc000097f48, 0xc000097ef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc000097e90 sp=0xc000097d50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003436b40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc000097fc8 sp=0xc000097e90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc000097fe0 sp=0xc000097fc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000097fe8 sp=0xc000097fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 488342 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x23b?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c67af58 sp=0xc00c67af38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f243100?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00c67af90 sp=0xc00c67af58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f243018, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00c67afb0 sp=0xc00c67af90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00581ba00?, 0xc000d56000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00c67afd8 sp=0xc00c67afb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00581ba00, {0xc000d56000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00c67b058 sp=0xc00c67afd8 pc=0x535efa net.(*netFD).Read(0xc00581ba00, {0xc000d56000?, 0x7b5a68?, 0xc00581ba00?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00c67b0a0 sp=0xc00c67b058 pc=0x7a20c9 net.(*conn).Read(0xc004081338, {0xc000d56000?, 0x164290f?, 0xc004081338?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00c67b0e8 sp=0xc00c67b0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03be73440, {0xc000d56000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00c67b130 sp=0xc00c67b0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc013dcdc08, {0xc000d56000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00c67b1d0 sp=0xc00c67b130 pc=0x370f896 bufio.(*Reader).fill(0xc0077e6090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00c67b208 sp=0xc00c67b1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0077e6090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00c67b228 sp=0xc00c67b208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0077e6668?, {0x62295c8, 0xc0077e6090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00c67b260 sp=0xc00c67b228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0077e6000, 0xc0174ebc20, 0xc00c67b442, 0xc00c67b444, {0x623b718, 0xc0153c21b0}, 0xc0059fa000, 0xc010b14c74, 0xc00c67b440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00c67b398 sp=0xc00c67b260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0077e6000, {0x624a948, 0xc002afd800}, 0xc0174ebc10, 0xc0059fa000, 0xc002afcff0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00c67b768 sp=0xc00c67b398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc002afd800}, {0x628a538, 0xc03be73440}, {{{0xc000d5a038, 0x4}}, 0x1, 0xc002afd110, 0xc002afd230, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00c67b8a8 sp=0xc00c67b768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc002afd800}, {0x628a538?, 0xc03be73440}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00c67be20 sp=0xc00c67b8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc002afce40?}, {0x628a538, 0xc03be73440}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00c67beb0 sp=0xc00c67be20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc002afce40}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00c67bf30 sp=0xc00c67beb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c67bfe0 sp=0xc00c67bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c67bfe8 sp=0xc00c67bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 475525 [sleep]: runtime.gopark(0x4b90df85c95?, 0x8019ba?, 0xf3?, 0xac?, 0x80f6c10351?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00efc3df0 sp=0xc00efc3dd0 pc=0x4907d6 time.Sleep(0x144db0ed5) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00efc3e30 sp=0xc00efc3df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc011cea0f0, 0x8019ba, {0x4618e7?, 0x203004?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00efc3e88 sp=0xc00efc3e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc011cea0f0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00efc3ec8 sp=0xc00efc3e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00e3e0f00?, 0x9, 0xc0000947b8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00efc3fa8 sp=0xc00efc3ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00efc3fe0 sp=0xc00efc3fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00efc3fe8 sp=0xc00efc3fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 446796 [sleep]: runtime.gopark(0x4b8ae597e35?, 0x320c1f?, 0xcf?, 0x29?, 0x8089afb62c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a447df0 sp=0xc00a447dd0 pc=0x4907d6 time.Sleep(0x13768a261) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a447e30 sp=0xc00a447df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc015ac5920, 0x320c1f, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a447e88 sp=0xc00a447e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc015ac5920, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a447ec8 sp=0xc00a447e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00187ef80?, {0xc014c41400?, 0x13, 0xc00187ef70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a447fa8 sp=0xc00a447ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a447fe0 sp=0xc00a447fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a447fe8 sp=0xc00a447fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 1045 [runnable]: runtime.gopark(0xc0036b8d80?, 0xb?, 0x0?, 0x0?, 0x8ff?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0036b8a80 sp=0xc0036b8a60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x40?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0036b8ab8 sp=0xc0036b8a80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd258, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0036b8ad8 sp=0xc0036b8ab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00091c700?, 0xc003d7c000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0036b8b00 sp=0xc0036b8ad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00091c700, {0xc003d7c000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0036b8b80 sp=0xc0036b8b00 pc=0x535efa net.(*netFD).Read(0xc00091c700, {0xc003d7c000?, 0x50c7cc0?, 0x4cd801?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0036b8bc8 sp=0xc0036b8b80 pc=0x7a20c9 net.(*conn).Read(0xc00762a238, {0xc003d7c000?, 0x100c003d7c000?, 0xc011b13f80?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0036b8c10 sp=0xc0036b8bc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc003ca2ba0, {0xc00179eac0, 0x9, 0xc0040c6dd4?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc0036b8c48 sp=0xc0036b8c10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc003ca2ba0}, {0xc00179eac0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc0036b8c90 sp=0xc0036b8c48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc00179eac0?, 0x9?, 0xc011b13ef0?}, {0x62052c0?, 0xc003ca2ba0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc0036b8ce0 sp=0xc0036b8c90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00179ea80) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc0036b8d90 sp=0xc0036b8ce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003436b40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc0036b8fc8 sp=0xc0036b8d90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc0036b8fe0 sp=0xc0036b8fc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0036b8fe8 sp=0xc0036b8fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 3725 [select, 9 minutes]: runtime.gopark(0xc006979e58?, 0x2?, 0x0?, 0x30?, 0xc006979dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006979c80 sp=0xc006979c60 pc=0x4907d6 runtime.selectgo(0xc006979e58, 0xc006979df8, 0xc001f7bb00?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006979dc0 sp=0xc006979c80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc001f7bb00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc006979f58 sp=0xc006979dc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc006979f78 sp=0xc006979f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc006979fe0 sp=0xc006979f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006979fe8 sp=0xc006979fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1046 [runnable]: runtime.Gosched(...) GOROOT/src/runtime/proc.go:318 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc003ca2c00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:570 +0x1f6 fp=0xc006339f88 sp=0xc006339f30 pc=0xbd7156 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc006339fe0 sp=0xc006339f88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006339fe8 sp=0xc006339fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 3938 [select]: runtime.gopark(0xc006a42f70?, 0x4?, 0x6?, 0x0?, 0xc006a42ed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a42d18 sp=0xc006a42cf8 pc=0x4907d6 runtime.selectgo(0xc006a42f70, 0xc006a42ec8, 0x1d73f8d?, 0x0, 0x624a948?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006a42e58 sp=0xc006a42d18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0038e3380) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc006a42fc8 sp=0xc006a42e58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc006a42fe0 sp=0xc006a42fc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a42fe8 sp=0xc006a42fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 1162 [select]: runtime.gopark(0xc017547ec0?, 0x2?, 0x5c?, 0x0?, 0xc017547e74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ec95cd0 sp=0xc00ec95cb0 pc=0x4907d6 runtime.selectgo(0xc00ec95ec0, 0xc017547e70, 0xc004770300?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ec95e10 sp=0xc00ec95cd0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc000b71570, {0x624a8a0, 0xc004770300}) github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:251 +0x18f fp=0xc00ec95f08 sp=0xc00ec95e10 pc=0x39372af github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop-fm({0x624a948?, 0xc00449b860?}) :1 +0x39 fp=0xc00ec95f30 sp=0xc00ec95f08 pc=0x3938379 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ec95fe0 sp=0xc00ec95f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ec95fe8 sp=0xc00ec95fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 428332 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00ce45640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0161075d0 sp=0xc0161075b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00f99e490, 0x139) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc016107618 sp=0xc0161075d0 pc=0x4be30c sync.(*Cond).Wait(0xc0100d1800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc016107650 sp=0xc016107618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0100d1e68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc016107708 sp=0xc016107650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00404e600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc016107cb8 sp=0xc016107708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00404e600, {0x624a8a0, 0xc00f99e4c0}, 0x41191a8c00000000?, 0xc003f55dc0?, 0xc010e21470?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc016107d70 sp=0xc016107cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00f99e4c0?}, {0xc0059fa000?}, 0x3?, 0xc0052a1a10?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc016107e20 sp=0xc016107d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc016107fe0 sp=0xc016107e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016107fe8 sp=0xc016107fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 1161 [select, 9 minutes]: runtime.gopark(0xc0032dd6d0?, 0x2?, 0x30?, 0x0?, 0xc0032dd6ac?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032dd528 sp=0xc0032dd508 pc=0x4907d6 runtime.selectgo(0xc0032dd6d0, 0xc0032dd6a8, 0xc00449b800?, 0x0, 0x1dd022d?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032dd668 sp=0xc0032dd528 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc00a1b82d0, {0x624a948?, 0xc00449b800?}) github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:354 +0x170 fp=0xc0032dd708 sp=0xc0032dd668 pc=0x393ab90 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop-fm({0x624a948?, 0xc00449b800?}) :1 +0x39 fp=0xc0032dd730 sp=0xc0032dd708 pc=0x393cb59 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0032dd7e0 sp=0xc0032dd730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032dd7e8 sp=0xc0032dd7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49852 [select, 7 minutes]: runtime.gopark(0xc00a56bfb0?, 0x2?, 0x80?, 0x46?, 0xc00a56bf94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003eade18 sp=0xc003eaddf8 pc=0x4907d6 runtime.selectgo(0xc003eadfb0, 0xc00a56bf90, 0x53ddbcd?, 0x0, 0xc00a56bfb8?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003eadf58 sp=0xc003eade18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc003eadfe0 sp=0xc003eadf58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003eadfe8 sp=0xc003eadfe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 485575 [sleep]: runtime.gopark(0x4b926a616ae?, 0x1002d25?, 0x87?, 0x84?, 0x810b8ca4ed?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d881df0 sp=0xc00d881dd0 pc=0x4907d6 time.Sleep(0x140f8f734) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00d881e30 sp=0xc00d881df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc006965020, 0x1002d25, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00d881e88 sp=0xc00d881e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc006965020, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00d881ec8 sp=0xc00d881e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc003548360?, {0xc02035f900?, 0x9, 0x0?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00d881fa8 sp=0xc00d881ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00d881fe0 sp=0xc00d881fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d881fe8 sp=0xc00d881fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498486 [select]: runtime.gopark(0xc005116c78?, 0x3?, 0x4?, 0x30?, 0xc005116a9e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005116900 sp=0xc0051168e0 pc=0x4907d6 runtime.selectgo(0xc005116c78, 0xc005116a98, 0x1?, 0x0, 0xc00564da10?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005116a40 sp=0xc005116900 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForQuery(0xc004959320, {0x624a948, 0xc00564da10}, 0xc007347d40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:805 +0x585 fp=0xc005116cb8 sp=0xc005116a40 pc=0x19bb485 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc00b56cf00, {0x624a948, 0xc00564da10}, {0x0, {0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:358 +0x175 fp=0xc005116cf8 sp=0xc005116cb8 pc=0x1cbd995 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc00564da10}, 0xc047f7ad20, 0xc005117298?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc005117168 sp=0xc005116cf8 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00564da10}, 0xc005117350?, {0x0, {0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0051172a8 sp=0xc005117168 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00564da10}, 0xc0172270e0, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0051175c8 sp=0xc0051172a8 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc005117990 sp=0xc0051175c8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc005118318 sp=0xc005117990 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x9c5fde0?, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc005118560 sp=0xc005118318 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00564d9b0}, {0x9172428?}, 0xc017226ff0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc005118bd0 sp=0xc005118560 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00564d800}, 0xc017226ff0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc005118c78 sp=0xc005118bd0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00564d800?}, {0x52f0080?, 0xc017226ff0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc005118ca8 sp=0xc005118c78 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00564d800}, {0x52f0080, 0xc017226ff0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc005118fd8 sp=0xc005118ca8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d800?}, {0x52f0080?, 0xc017226ff0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc005119018 sp=0xc005118fd8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00564d800}, {0x52f0080, 0xc017226ff0}, 0xc00c88d0b0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc005119048 sp=0xc005119018 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d800?}, {0x52f0080?, 0xc017226ff0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc005119088 sp=0xc005119048 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00564d800?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0051190c0 sp=0xc005119088 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00564d800}, {0x203004?, 0xb7fc1f1b00203004?}, 0xc00c88d178) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc005119138 sp=0xc0051190c0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00564d800?}, {0x52f0080?, 0xc017226ff0?}, 0xc00c88d218?, 0xc00c88d238?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0051191b0 sp=0xc005119138 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d800?}, {0x52f0080?, 0xc017226ff0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0051191f0 sp=0xc0051191b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00564d800?}, {0xc00564d800?, 0x4?}, {0x52f0080?, 0xc017226ff0?}, {0x51b9a20?, 0xc0533f6280}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc005119230 sp=0xc0051191f0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00564d800}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017226ff0}, {0x51b9a20, 0xc0533f6280}, 0xc00c88d3a8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc005119320 sp=0xc005119230 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00564d800}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017226ff0}, {0x51b9a20, 0xc0533f6280}, 0xc00c88d488?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc0051193b8 sp=0xc005119320 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00564d770}, 0xc017226ff0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc005119438 sp=0xc0051193b8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc002698400?, {0x624a948?, 0xc00564d770?}, 0xc00c88d4a8?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc0051194c0 sp=0xc005119438 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0113e85a0, {_, _}, _, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc005119760 sp=0xc0051194c0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0113e85a0, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc005119898 sp=0xc005119760 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc005119fe0 sp=0xc005119898 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00511a8e0 sp=0xc005119fe0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00511b148 sp=0xc00511a8e0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x17600a76e48c96ba, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00511b568 sp=0xc00511b148 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00511b6a0 sp=0xc00511b568 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00511b7c8 sp=0xc00511b6a0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0x0?, {_, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc00511b8e8 sp=0xc00511b7c8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e48c7bd7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00511b9f8 sp=0xc00511b8e8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc00564d740}, 0x0?, 0xc0174ff080) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00511bbf0 sp=0xc00511b9f8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc00564d740}, 0x3fc3333333333333?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc00511bc38 sp=0xc00511bbf0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc004959320, {0x624a948, 0xc00564d740}, {{0x25, 0x72, 0x4d, 0x79, 0x3c, 0xd5, 0x42, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc00511bc88 sp=0xc00511bc38 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).startQueryPusherTxn.func1({0x624a948, 0xc00564d740}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:854 +0x238 fp=0xc00511bf30 sp=0xc00511bc88 pc=0x19bbe78 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00511bfe0 sp=0xc00511bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00511bfe8 sp=0xc00511bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1181 [select]: runtime.gopark(0xc008a51ee0?, 0x3?, 0x8d?, 0x0?, 0xc008a51ea2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008a51d28 sp=0xc008a51d08 pc=0x4907d6 runtime.selectgo(0xc008a51ee0, 0xc008a51e9c, 0xc0041069f0?, 0x0, 0x10000c0011ea7b8?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008a51e68 sp=0xc008a51d28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2({0x624a948, 0xc0041069f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:251 +0x1b7 fp=0xc008a51f30 sp=0xc008a51e68 pc=0x1d1adb7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc008a51fe0 sp=0xc008a51f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008a51fe8 sp=0xc008a51fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1074 [select, 9 minutes]: runtime.gopark(0xc0032e1fb0?, 0x2?, 0x20?, 0x73?, 0xc0032e1f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032e1e18 sp=0xc0032e1df8 pc=0x4907d6 runtime.selectgo(0xc0032e1fb0, 0xc0032e1f90, 0xc0047af880?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032e1f58 sp=0xc0032e1e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0032e1fe0 sp=0xc0032e1f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032e1fe8 sp=0xc0032e1fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 1075 [select, 9 minutes]: runtime.gopark(0xc0032e17b0?, 0x2?, 0xb0?, 0x5c?, 0xc0032e179c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032e1628 sp=0xc0032e1608 pc=0x4907d6 runtime.selectgo(0xc0032e17b0, 0xc0032e1798, 0xc002a40360?, 0x0, 0x140703?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032e1768 sp=0xc0032e1628 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0032e17e0 sp=0xc0032e1768 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032e17e8 sp=0xc0032e17e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 1076 [select, 9 minutes]: runtime.gopark(0xc006a40b70?, 0x2?, 0x0?, 0x0?, 0xc006a40b6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a409f8 sp=0xc006a409d8 pc=0x4907d6 runtime.selectgo(0xc006a40b70, 0xc006a40b68, 0xc006a40ba8?, 0x0, 0xc001395c30?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006a40b38 sp=0xc006a409f8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc006bf79e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc006a40ba0 sp=0xc006a40b38 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ca51e0, {0x512dba0?, 0xc003cc8f00}, 0xc0001a0000?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc006a40ca8 sp=0xc006a40ba0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc006a40cd8 sp=0xc006a40ca8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc006bf7320, 0xc0032dddb0, 0xc0032ddd80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc006a40d30 sp=0xc006a40cd8 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc006bf7320, {0x512dba0?, 0xc003cc8f00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc006a40de0 sp=0xc006a40d30 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0039e7500, {0x512dba0?, 0xc003cc8f00?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc006a40e28 sp=0xc006a40de0 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc005ec6310) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:132 +0x4c fp=0xc006a40e58 sp=0xc006a40e28 pc=0x1d4b00c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x628a900, 0xc005ec6310}, 0xc000c62700, {0x624a948, 0xc003cc8ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:409 +0x5b fp=0xc006a40ee0 sp=0xc006a40e58 pc=0x1d843fb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x624a948?, 0xc003cc8ea0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:424 +0x45 fp=0xc006a40f30 sp=0xc006a40ee0 pc=0x1d84345 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006a40fe0 sp=0xc006a40f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a40fe8 sp=0xc006a40fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1164 [select, 9 minutes]: runtime.gopark(0xc003dcfef0?, 0x3?, 0x1?, 0x30?, 0xc003dcfe52?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d90c98 sp=0xc002d90c78 pc=0x4907d6 runtime.selectgo(0xc002d90ef0, 0xc003dcfe4c, 0x0?, 0x0, 0x12ea4647f?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d90dd8 sp=0xc002d90c98 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1({0x624a948, 0xc00449b8c0}) github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:628 +0x2af fp=0xc002d90f30 sp=0xc002d90dd8 pc=0x3515aaf github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002d90fe0 sp=0xc002d90f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d90fe8 sp=0xc002d90fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1067 [select]: runtime.gopark(0xc0085bdca0?, 0x5?, 0x5?, 0x0?, 0xc0085bda66?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0085bd898 sp=0xc0085bd878 pc=0x4907d6 runtime.selectgo(0xc0085bdca0, 0xc0085bda5c, 0xc003e9a840?, 0x0, 0x17600a611d9f5ae4?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0085bd9d8 sp=0xc0085bd898 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc00742cf30, {0x624a8a0, 0xc003e9a840}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:323 +0x95f fp=0xc0085bdd60 sp=0xc0085bd9d8 pc=0x326315f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x624a948?, 0xc003d67a70?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x27a fp=0xc0085bdf30 sp=0xc0085bdd60 pc=0x326255a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0085bdfe0 sp=0xc0085bdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0085bdfe8 sp=0xc0085bdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1068 [select, 9 minutes]: runtime.gopark(0xc0032ea698?, 0x2?, 0x1?, 0x30?, 0xc0032ea5ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032ea428 sp=0xc0032ea408 pc=0x4907d6 runtime.selectgo(0xc0032ea698, 0xc0032ea5e8, 0xffffffffffffffff?, 0x0, 0x1e540f5?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032ea568 sp=0xc0032ea428 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/server.(*Server).startSystemLogsGC.func1({0x624a948, 0xc003d67ad0}) github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:179 +0x165 fp=0xc0032ea730 sp=0xc0032ea568 pc=0x39e3205 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0032ea7e0 sp=0xc0032ea730 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032ea7e8 sp=0xc0032ea7e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1033 [semacquire, 9 minutes]: runtime.gopark(0xc00605ba68?, 0x46159f?, 0xa0?, 0xda?, 0xc003f80f30?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003f519f0 sp=0xc003f519d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc001e2f490, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc003f51a58 sp=0xc003f519f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc003f51a88 sp=0xc003f51a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc00605bae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc003f51ab0 sp=0xc003f51a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc001e2f480) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc003f51ac8 sp=0xc003f51ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc001e2f480?, {0x624a8a0?, 0xc001e2f440?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc003f51af8 sp=0xc003f51ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc0035f9d40?}, {0xc00605bc68, 0x2, 0xc00605bdc0?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc003f51b48 sp=0xc003f51af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0016c22c0, {0x624a948, 0xc0035f9d40}, 0xc003e9a8c0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc003f51e68 sp=0xc003f51b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc0035f9cb0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc003f51f30 sp=0xc003f51e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003f51fe0 sp=0xc003f51f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003f51fe8 sp=0xc003f51fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 67292 [select]: runtime.gopark(0xc00aaf4f20?, 0x4?, 0x0?, 0x0?, 0xc00aaf4da0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00aaf4bf0 sp=0xc00aaf4bd0 pc=0x4907d6 runtime.selectgo(0xc00aaf4f20, 0xc00aaf4d98, 0xc00dbf82d0?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00aaf4d30 sp=0xc00aaf4bf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher.(*SQLWatcher).watch(0xc0059229b0, {0x624a948, 0xc00dbf82d0}, {0x18?, 0x4e1f660?, 0x0?}, 0xc0131308b8) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher/sqlwatcher.go:160 +0x665 fp=0xc00aaf4f70 sp=0xc00aaf4d30 pc=0x392b685 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher.(*SQLWatcher).WatchForSQLUpdates(0xc0131308a0?, {0x624a948?, 0xc00dbf82d0?}, {0xc00b956ff0?, 0x4618e7?, 0x0?}, 0x18?) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher/sqlwatcher.go:90 +0x25 fp=0xc00aaf4fb0 sp=0xc00aaf4f70 pc=0x392afc5 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler.(*incrementalReconciler).reconcile(0xc008d17dc0, {0x624a948, 0xc00dbf82d0}, {0xc00765ddc0?, 0xad55038?, 0xc0?}, 0xc0131308a0) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler/reconciler.go:454 +0xb7 fp=0xc00aaf5000 sp=0xc00aaf4fb0 pc=0x39258f7 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler.(*Reconciler).Reconcile(0xc00597d200, {0x624a948, 0xc00dbf82d0}, {0x6fc23ac00?, 0xcccccccd?, 0xcc?}, {0x6267438, 0xc004738a80}, 0xc00765ddc0) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler/reconciler.go:187 +0x3f3 fp=0xc00aaf5138 sp=0xc00aaf5000 pc=0x39230b3 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob.(*resumer).Resume(0xc00ce6d178, {0x624a948?, 0xc00dbf82d0}, {0x50d3280?, 0xc00ce6d1b0}) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob/job.go:162 +0xb88 fp=0xc00aaf5598 sp=0xc00aaf5138 pc=0x376b8e8 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2(0x624a948?, 0x624a948?, {0x62219e8?, 0xc00ce6d178?}, {0x624a948?, 0xc00dbf82d0?}, {0x50d3280?, 0xc00ce6d1b0?}, 0xc00aaf5870) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1262 +0xcc fp=0xc00aaf55f8 sp=0xc00aaf5598 pc=0x22a10ac github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine(0xc00a1b5600, {0x624a948, 0xc00d4692f0}, {0x50d3280, 0xc00ce6d1b0}, {0x62219e8, 0xc00ce6d178}, 0xc00d549380, {0xc00feca1c0, 0x7}, ...) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1263 +0x8ff fp=0xc00aaf5ca8 sp=0xc00aaf55f8 pc=0x229f75f github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob(0xc00a1b5600, {0x624a948, 0xc00d469080}, {0x62219e8, 0xc00ce6d178}, 0xc00d549380, {0xc00feca1c0, 0x7}, {0xc004c9ebe8, 0x16}) github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:410 +0x5c6 fp=0xc00aaf5e80 sp=0xc00aaf5ca8 pc=0x2287c06 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func1({0xc006d26000?, 0xc006a558f0?}) github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:330 +0xfa fp=0xc00aaf5f30 sp=0xc00aaf5e80 pc=0x228739a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00aaf5fe0 sp=0xc00aaf5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00aaf5fe8 sp=0xc00aaf5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498013 [sleep]: runtime.gopark(0x4b91ee8a427?, 0x1002f26?, 0x0?, 0xdb?, 0x810417fb53?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00394bdf0 sp=0xc00394bdd0 pc=0x4907d6 time.Sleep(0x14141c2a8) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00394be30 sp=0xc00394bdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc002ed29c0, 0x1002f26, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00394be88 sp=0xc00394be30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc002ed29c0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00394bec8 sp=0xc00394be88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc02a00e280?, 0x4, 0xc00845a180?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00394bfa8 sp=0xc00394bec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00394bfe0 sp=0xc00394bfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00394bfe8 sp=0xc00394bfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 3756 [select]: runtime.gopark(0xc00b607ef8?, 0x2?, 0x90?, 0x7d?, 0xc00b607ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b5edd58 sp=0xc00b5edd38 pc=0x4907d6 runtime.selectgo(0xc00b5edef8, 0xc00b607ed0, 0xc0028e7aa0?, 0x0, 0xc00143bf00?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b5ede98 sp=0xc00b5edd58 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1({0x624a948, 0xc0028e7aa0}) github.com/cockroachdb/cockroach/pkg/ts/db.go:176 +0xfb fp=0xc00b5edf30 sp=0xc00b5ede98 pc=0x35b191b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b5edfe0 sp=0xc00b5edf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b5edfe8 sp=0xc00b5edfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1115 [select]: runtime.gopark(0xc001837e38?, 0x3?, 0x0?, 0x0?, 0xc001837dea?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001837c40 sp=0xc001837c20 pc=0x4907d6 runtime.selectgo(0xc001837e38, 0xc001837de4, 0xc007814890?, 0x0, 0x4618e7?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001837d80 sp=0xc001837c40 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00370a380, {0x624a8a0, 0xc0043b15c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:322 +0x265 fp=0xc001837e78 sp=0xc001837d80 pc=0x1c8b565 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00370a380, {0x624a948, 0xc004817620}, 0xc0040a3ad0?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:347 +0xc5 fp=0xc001837eb0 sp=0xc001837e78 pc=0x1c8b7c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x624a948?, 0xc004817620?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:296 +0x4c fp=0xc001837f30 sp=0xc001837eb0 pc=0x1c86a8c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001837fe0 sp=0xc001837f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001837fe8 sp=0xc001837fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1163 [select, 3 minutes]: runtime.gopark(0xc006a43eb0?, 0x4?, 0xd0?, 0x46?, 0xc006a43e68?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006a43cd8 sp=0xc006a43cb8 pc=0x4907d6 runtime.selectgo(0xc006a43eb0, 0xc006a43e60, 0xc007244710?, 0x0, 0xc003de7800?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006a43e18 sp=0xc006a43cd8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc0058d65a0, {0x48b?, 0xc000ba0f00?}) github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:166 +0x2a7 fp=0xc006a43f08 sp=0xc006a43e18 pc=0x326fbe7 github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run-fm({0x624a948?, 0xc00449b890?}) :1 +0x39 fp=0xc006a43f30 sp=0xc006a43f08 pc=0x32702b9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006a43fe0 sp=0xc006a43f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006a43fe8 sp=0xc006a43fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 467470 [sleep]: runtime.gopark(0x4b8a8d641d6?, 0x40466?, 0xd3?, 0xdc?, 0x8086309f28?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00394fdf0 sp=0xc00394fdd0 pc=0x4907d6 time.Sleep(0x1396cc719) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00394fe30 sp=0xc00394fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0083e2a80, 0x40466, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00394fe88 sp=0xc00394fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0083e2a80, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00394fec8 sp=0xc00394fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc008ca2500?, 0x1c, 0xc004ee7f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00394ffa8 sp=0xc00394fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00394ffe0 sp=0xc00394ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00394ffe8 sp=0xc00394ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 45299 [select]: runtime.gopark(0xc010259db8?, 0x3?, 0x76?, 0x0?, 0xc010259d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010259b90 sp=0xc010259b70 pc=0x4907d6 runtime.selectgo(0xc010259db8, 0xc010259d2c, 0xc00db4e180?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc010259cd0 sp=0xc010259b90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc00db4e180}, 0xc007cfd310, {0xc005e76bb8, 0x12}, 0xc002d107e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc010259e68 sp=0xc010259cd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc00db4e180}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc010259f30 sp=0xc010259e68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010259fe0 sp=0xc010259f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010259fe8 sp=0xc010259fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1121 [semacquire, 9 minutes]: runtime.gopark(0xc002241a68?, 0x46159f?, 0xe0?, 0x39?, 0xc003eddb00?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006ae79f0 sp=0xc006ae79d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc001ad9c90, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc006ae7a58 sp=0xc006ae79f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc006ae7a88 sp=0xc006ae7a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc002241ae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc006ae7ab0 sp=0xc006ae7a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc001ad9c80) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc006ae7ac8 sp=0xc006ae7ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc001ad9c80?, {0x624a8a0?, 0xc001ad9c40?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc006ae7af8 sp=0xc006ae7ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc000ac1d70?}, {0xc002241c68, 0x2, 0xc002241dc0?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc006ae7b48 sp=0xc006ae7af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001786000, {0x624a948, 0xc000ac1d70}, 0xc0009ca780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc006ae7e68 sp=0xc006ae7b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc000ac1cb0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc006ae7f30 sp=0xc006ae7e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006ae7fe0 sp=0xc006ae7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006ae7fe8 sp=0xc006ae7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1120 [select]: runtime.gopark(0xc000057ca0?, 0x5?, 0x0?, 0x0?, 0xc000057a66?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000057898 sp=0xc000057878 pc=0x4907d6 runtime.selectgo(0xc000057ca0, 0xc000057a5c, 0xc0009ca600?, 0x0, 0xc0009ca600?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0000579d8 sp=0xc000057898 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc00742cea0, {0x624a8a0, 0xc0009ca600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:323 +0x95f fp=0xc000057d60 sp=0xc0000579d8 pc=0x326315f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x624a948?, 0xc000ac0150?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x27a fp=0xc000057f30 sp=0xc000057d60 pc=0x326255a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc000057fe0 sp=0xc000057f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000057fe8 sp=0xc000057fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3963 [select]: runtime.gopark(0xc001b2ef70?, 0x4?, 0x6?, 0x0?, 0xc001b2eed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b2ed18 sp=0xc001b2ecf8 pc=0x4907d6 runtime.selectgo(0xc001b2ef70, 0xc001b2eec8, 0x140da7532?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001b2ee58 sp=0xc001b2ed18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc004caa680) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc001b2efc8 sp=0xc001b2ee58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc001b2efe0 sp=0xc001b2efc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b2efe8 sp=0xc001b2efe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 484229 [sleep]: runtime.gopark(0x4b8a8a67332?, 0x1b9b603?, 0xab?, 0xee?, 0x808612b0fe?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a353df0 sp=0xc00a353dd0 pc=0x4907d6 time.Sleep(0x1397ea981) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a353e30 sp=0xc00a353df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc01618dc20, 0x1b9b603, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a353e88 sp=0xc00a353e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc01618dc20, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a353ec8 sp=0xc00a353e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc004b36600?, {0xc01fa9abe0?, 0x4, 0xc013d34720?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a353fa8 sp=0xc00a353ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a353fe0 sp=0xc00a353fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a353fe8 sp=0xc00a353fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498495 [runnable]: runtime.gopark(0xae1b643770d?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00adf05f8 sp=0xc00adf05d8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00adf0660 sp=0xc00adf05f8 pc=0x4a17de sync.runtime_SemacquireMutex(0x0?, 0xa0?, 0xc003846c60?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00adf0690 sp=0xc00adf0660 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0xc001686dc0?, 0xe?, 0x50?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00adf06e8 sp=0xc00adf0690 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00adf0730 sp=0xc00adf06e8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004e6b340, {0xc00adf0870?, 0x4ccece?}, 0x0, 0x0, 0xc02135fc10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00adf0830 sp=0xc00adf0730 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00da38900, 0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc00adf0880 sp=0xc00adf0830 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc02135fc10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc00adf0908 sp=0xc00adf0880 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc002941970, 0x15f2364?, 0xc021360b40, {0x0?, 0xc02135fc10?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc00adf0a38 sp=0xc00adf0908 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x7f258149ad74?, 0x0?, 0xc001686dc0?, {0x0?, 0xc02135fc10?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc00adf0a88 sp=0xc00adf0a38 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc021360af0?, 0xc005abd080?, {0x0?, 0xc02135fc10?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc00adf0ad0 sp=0xc00adf0a88 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc021360af0, 0xc005abd080, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc00adf0b30 sp=0xc00adf0ad0 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc021360af0, {0xc001686dc0, 0x6, 0x50}, {0xc010cb9190, 0x6, 0x6}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc00adf0b80 sp=0xc00adf0b30 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc02135fe20, {0xc010cb9190?, 0xc003357828?, 0x1000000000000?}, 0x7f275fc7a458?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00adf0cb0 sp=0xc00adf0b80 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc02135fe20, {0xc001686dc0?, 0x0?, 0x0?}, {0xc010cb9190?, 0x4a59d2?, 0x6?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00adf0cf0 sp=0xc00adf0cb0 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc02135fc88, {0xc001686dc0?, 0x0?, 0x6?}, {0xc010cb9190, 0x6, 0x6}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00adf0d48 sp=0xc00adf0cf0 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc02135f900, {0xc010cb9190, 0x6, 0x6}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00adf0de0 sp=0xc00adf0d48 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc003357828, {{0xc01121c818, 0x5, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00adf0e30 sp=0xc00adf0de0 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekIntentGE(0xc0051f3c00, {0xc01121c818, 0x5, 0x8}, {0x4f, 0x68, 0xbe, 0x4f, 0x3e, 0xf0, ...}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:540 +0x117 fp=0xc00adf0f58 sp=0xc00adf0e30 pc=0x180d357 github.com/cockroachdb/cockroach/pkg/storage.MVCCResolveWriteIntent({0x624a948, 0xc00aeb2540}, {0x7f275f2b7158?, 0xc003357800?}, 0xc000580000?, {{{0xc01121c818, 0x5, 0x8}, {0x0, 0x0, ...}}, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3968 +0x12b fp=0xc00adf10b0 sp=0xc00adf0f58 pc=0x182dfab github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.ResolveIntent({_, _}, {_, _}, {{0x62cc4b0, 0xc012e422a0}, {{0x17600a76e49d883d, 0x0, 0x0}, 0xc010cb9180, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_resolve_intent.go:96 +0x385 fp=0xc00adf16e8 sp=0xc00adf10b0 pc=0x1d02d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e49d883d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc00adf1f78 sp=0xc00adf16e8 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc017227590, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00adf3de8 sp=0xc00adf1f78 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00ed130e0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00adf4438 sp=0xc00adf3de8 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00ed130e0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00adf4910 sp=0xc00adf4438 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a77026aed3d, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00adf5790 sp=0xc00adf4910 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc003440800?, {0x624a948, 0xc00aeb2540}, {0xc010cb9178, 0x8}, 0xc017227590, 0x8?, 0x0?, {{0x17600a77026aed3d, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00adf6410 sp=0xc00adf5790 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc00aeb2540}, {0xc010cb9178, 0x8}, 0xc017227590, 0x400000004?, 0xc00adf7460, {{0x17600a77026aed3d, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00adf64c8 sp=0xc00adf6410 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc003861800, {0x624a948, 0xc00aeb2540}, 0xc017227590, 0xc00ed130e0, 0xc00adf7460, {{0x17600a77026aed3d, 0x0, 0x0}, {0x17600a76e49d883d, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00adf6e90 sp=0xc00adf64c8 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc00aeb2540}, 0xc017227590, 0xc00ed130e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00adf7600 sp=0xc00adf6e90 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00aeb2540}, 0xc017227590, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00adf7920 sp=0xc00adf7600 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e49d883d, 0x0, 0x0}, 0xc010cb9180, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00adf7ce8 sp=0xc00adf7920 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e49d883d, 0x0, 0x0}, 0xc010cb9180, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00adf8670 sp=0xc00adf7ce8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x624a948?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00adf88b8 sp=0xc00adf8670 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00aeb24e0}, {0xc006a4a780?}, 0xc0172274a0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00adf8f28 sp=0xc00adf88b8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00aeb2450}, 0xc0172274a0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00adf8fd0 sp=0xc00adf8f28 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00aeb2450?}, {0x52f0080?, 0xc0172274a0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00adf9000 sp=0xc00adf8fd0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00aeb2450}, {0x52f0080, 0xc0172274a0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00adf9330 sp=0xc00adf9000 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00aeb2450?}, {0x52f0080?, 0xc0172274a0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00adf9370 sp=0xc00adf9330 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00aeb2450}, {0x52f0080, 0xc0172274a0}, 0xc00c88d408?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00adf93a0 sp=0xc00adf9370 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00aeb2450?}, {0x52f0080?, 0xc0172274a0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00adf93e0 sp=0xc00adf93a0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00aeb2450?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00adf9418 sp=0xc00adf93e0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00aeb2450}, {0xc003ca8000?, 0x3?}, 0xc00c88d4d0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00adf9490 sp=0xc00adf9418 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00aeb2450?}, {0x52f0080?, 0xc0172274a0?}, 0x51b9a20?, 0xc0533f6480?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00adf9508 sp=0xc00adf9490 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00aeb2450?}, {0x52f0080?, 0xc0172274a0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00adf9548 sp=0xc00adf9508 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00aeb2450?}, {0xc00aeb2450?, 0x4?}, {0x52f0080?, 0xc0172274a0?}, {0x51b9a20?, 0xc0533f6500}, 0x203002?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00adf9588 sp=0xc00adf9548 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb2450}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0172274a0}, {0x51b9a20, 0xc0533f6500}, 0xc00c88d700?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00adf9678 sp=0xc00adf9588 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00aeb2450}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0172274a0}, {0x51b9a20, 0xc0533f6500}, 0xc00c88d7e0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00adf9710 sp=0xc00adf9678 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00aeb22d0}, 0xc0172274a0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00adf9790 sp=0xc00adf9710 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc88d830?, {0x624a948?, 0xc00aeb22d0?}, 0x624a948?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00adf9818 sp=0xc00adf9790 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0103f88a0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00adf9ab8 sp=0xc00adf9818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0103f88a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00adf9bf0 sp=0xc00adf9ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00adfa338 sp=0xc00adf9bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00adfac38 sp=0xc00adfa338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00adfb4a0 sp=0xc00adfac38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49bd445, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00adfb8c0 sp=0xc00adfb4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00adfb9f8 sp=0xc00adfb8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00aeb22a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00adfbc00 sp=0xc00adfb9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00aeb22a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00adfbf30 sp=0xc00adfbc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00adfbfe0 sp=0xc00adfbf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00adfbfe8 sp=0xc00adfbfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3503 [select]: runtime.gopark(0xc008b0cd68?, 0x2?, 0x5?, 0x0?, 0xc008b0cd34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008b0cbb8 sp=0xc008b0cb98 pc=0x4907d6 runtime.selectgo(0xc008b0cd68, 0xc008b0cd30, 0x8?, 0x0, 0xc0231de000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008b0ccf8 sp=0xc008b0cbb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00491e5a0, {0xc001f7e328, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc008b0cd98 sp=0xc008b0ccf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00491e5a0, {0xc001f7e328?, 0xc0072f98d8?, 0xc008b0cde0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc008b0cdd8 sp=0xc008b0cd98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002b38510, {0xc001f7e328?, 0xc008b0ce58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc008b0ce20 sp=0xc008b0cdd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc002b38510}, {0xc001f7e328, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc008b0ce68 sp=0xc008b0ce20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0048d2d80, {0xc001f7e328, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc008b0cea8 sp=0xc008b0ce68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc001f7e318, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc008b0cf28 sp=0xc008b0cea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc007bc9028?, 0xc0048d2d80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc008b0cfc0 sp=0xc008b0cf28 pc=0xc19546 google.golang.org/grpc.recv(0x6?, {0x7f275f301038, 0xc0000af170}, 0xc2b369?, {0x0?, 0x0?}, {0x51d3160, 0xc00e1c5b30}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc008b0d038 sp=0xc008b0cfc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc004e33ad0, {0x51d3160?, 0xc00e1c5b30}, 0x5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc008b0d140 sp=0xc008b0d038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc008b0d170 sp=0xc008b0d140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0048d2a20, 0xc008b0d248, 0xc008b0d218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc008b0d1c8 sp=0xc008b0d170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0048d2a20, {0x51d3160?, 0xc00e1c5b30?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc008b0d278 sp=0xc008b0d1c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0039e7360, {0x51d3160?, 0xc00e1c5b30?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc008b0d2c0 sp=0xc008b0d278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc0008d9910) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc008b0d2f0 sp=0xc008b0d2c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc008b0d310 sp=0xc008b0d2f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00491e370, 0xc008b0d780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc008b0d3b8 sp=0xc008b0d310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc001cfe600?}, {{0xc0035cdb80, 0x1, 0x8}, {0xc0035cdb88, 0x1, 0x8}}, {0x176009f829266a9b, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc008b0d9a0 sp=0xc008b0d3b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc001cfe600}, 0xc004731d10, 0x624a948?, {{0xc0035cdb80, 0x1, 0x8}, {0xc0035cdb88, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc008b0de80 sp=0xc008b0d9a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc001cfe600?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc008b0df58 sp=0xc008b0de80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc008b0df78 sp=0xc008b0df58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc008b0dfe0 sp=0xc008b0df78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008b0dfe8 sp=0xc008b0dfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498450 [select]: runtime.gopark(0xc00b13b808?, 0x3?, 0x4?, 0x0?, 0xc00b13b4d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b13b258 sp=0xc00b13b238 pc=0x4907d6 runtime.selectgo(0xc00b13b808, 0xc00b13b4cc, 0x17600a765bd6a308?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b13b398 sp=0xc00b13b258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc00419fe00}, 0xc0081da0f0, 0xc0075780a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00b13bb08 sp=0xc00b13b398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc00419fe00}, 0xc0081da0f0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00b13be28 sp=0xc00b13bb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a7657c54cd9, 0x0, 0x0}, 0x0, {0x17600a76e1c8f896, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00b13c1f0 sp=0xc00b13be28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a7657c54cd9, 0x0, 0x0}, 0x0, {0x17600a76e1c8f896, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00b13cb78 sp=0xc00b13c1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1c8f896, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00b13cdc0 sp=0xc00b13cb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00419fbf0}, {0xc00b13d470?}, 0xc0081da000) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00b13d430 sp=0xc00b13cdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00419eea0}, 0xc0081da000) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00b13d4d8 sp=0xc00b13d430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc00419eea0}, {0x52f0080?, 0xc0081da000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00b13d518 sp=0xc00b13d4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00419eea0}, {0x52f0080, 0xc0081da000}, 0xc011dbb880, 0xc001c331b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00b13d848 sp=0xc00b13d518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00419eea0?}, {0x52f0080?, 0xc0081da000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00b13d888 sp=0xc00b13d848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00419eea0}, {0x52f0080, 0xc0081da000}, 0xc00b13d920?, 0xc00a1d7c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00b13d8b8 sp=0xc00b13d888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00419eea0?}, {0x52f0080?, 0xc0081da000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b13d8f8 sp=0xc00b13d8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00419eea0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00b13d930 sp=0xc00b13d8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00419eea0}, {0x203004?, 0x40?}, 0xc00b13d9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00b13d9a8 sp=0xc00b13d930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00419eea0?}, {0x52f0080?, 0xc0081da000?}, 0x5003b00?, 0x7f278cfaaa01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00b13da20 sp=0xc00b13d9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00419eea0?}, {0x52f0080?, 0xc0081da000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b13da60 sp=0xc00b13da20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc00419eea0}, {0x52f0080, 0xc0081da000}, 0xc011dbb880, 0xc001c331b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00b13dab0 sp=0xc00b13da60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc00419eea0}, 0xc010df5e60, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00b13db08 sp=0xc00b13dab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc010610c60, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00b13de48 sp=0xc00b13db08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc010610c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00b13df68 sp=0xc00b13de48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00b13dfe0 sp=0xc00b13df68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b13dfe8 sp=0xc00b13dfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 11306 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0xc99?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00de98f58 sp=0xc00de98f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f243790?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00de98f90 sp=0xc00de98f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f2436a8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00de98fb0 sp=0xc00de98f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc013663a80?, 0xc00d705000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00de98fd8 sp=0xc00de98fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc013663a80, {0xc00d705000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00de99058 sp=0xc00de98fd8 pc=0x535efa net.(*netFD).Read(0xc013663a80, {0xc00d705000?, 0x7b5a68?, 0xc013663a80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00de990a0 sp=0xc00de99058 pc=0x7a20c9 net.(*conn).Read(0xc005657368, {0xc00d705000?, 0x164290f?, 0xc005657368?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00de990e8 sp=0xc00de990a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002186e40, {0xc00d705000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00de99130 sp=0xc00de990e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00dbc22b8, {0xc00d705000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00de991d0 sp=0xc00de99130 pc=0x370f896 bufio.(*Reader).fill(0xc00e8d9c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00de99208 sp=0xc00de991d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00e8d9c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00de99228 sp=0xc00de99208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00e8da268?, {0x62295c8, 0xc00e8d9c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00de99260 sp=0xc00de99228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00e8d9c00, 0xc008e74700, 0xc00de99442, 0xc00de99444, {0x623b718, 0xc006afe7e0}, 0xc0059fa000, 0xc00d83cec4, 0xc00de99440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00de99398 sp=0xc00de99260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00e8d9c00, {0x624a948, 0xc001c56060}, 0xc008e746f0, 0xc0059fa000, 0xc001b3dc50, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00de99768 sp=0xc00de99398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc001c56060}, {0x628a538, 0xc002186e40}, {{{0xc003a29034, 0x4}}, 0x1, 0xc001b3dcb0, 0xc001b3dd70, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00de998a8 sp=0xc00de99768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc001c56060}, {0x628a538?, 0xc002186e40}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00de99e20 sp=0xc00de998a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc001b3daa0?}, {0x628a538, 0xc002186e40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00de99eb0 sp=0xc00de99e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc001b3daa0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00de99f30 sp=0xc00de99eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00de99fe0 sp=0xc00de99f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00de99fe8 sp=0xc00de99fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3798 [sync.Cond.Wait]: runtime.gopark(0xc00d1656d6?, 0xc0101a5c98?, 0x10?, 0x55?, 0xc01745b608?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ccd5c78 sp=0xc00ccd5c58 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc000b70b78, 0x1dff) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00ccd5cc0 sp=0xc00ccd5c78 pc=0x4be30c sync.(*Cond).Wait(0x5?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00ccd5cf8 sp=0xc00ccd5cc0 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000b70b60, {0x624a8a0, 0xc003745600}, 0xa18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb fp=0xc00ccd5db0 sp=0xc00ccd5cf8 pc=0x1d1cc5b github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x624a948?, 0xc0037fa600?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 fp=0xc00ccd5f30 sp=0xc00ccd5db0 pc=0x1d1dbd6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ccd5fe0 sp=0xc00ccd5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ccd5fe8 sp=0xc00ccd5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498221 [select]: runtime.gopark(0xc006711fa8?, 0x2?, 0x2?, 0x30?, 0xc006711f74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006711dc8 sp=0xc006711da8 pc=0x4907d6 runtime.selectgo(0xc006711fa8, 0xc006711f70, 0xc01379a030?, 0x0, 0x1054cca?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006711f08 sp=0xc006711dc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc006711fe0 sp=0xc006711f08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006711fe8 sp=0xc006711fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 3962 [select]: runtime.gopark(0xc005135f00?, 0x2?, 0x88?, 0x5d?, 0xc005135ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005135d60 sp=0xc005135d40 pc=0x4907d6 runtime.selectgo(0xc005135f00, 0xc005135ed0, 0x0?, 0x0, 0xc005135ef0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005135ea0 sp=0xc005135d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0049a9c20, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc005135f30 sp=0xc005135ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004c90300) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc005135f88 sp=0xc005135f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc005135fe0 sp=0xc005135f88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005135fe8 sp=0xc005135fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 1242 [semacquire, 9 minutes]: runtime.gopark(0xc0054afa68?, 0x46159f?, 0xc0?, 0x92?, 0xc004879cb0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0027399f0 sp=0xc0027399d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc001f7b790, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc002739a58 sp=0xc0027399f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc002739a88 sp=0xc002739a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc0054afae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc002739ab0 sp=0xc002739a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc001f7b780) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc002739ac8 sp=0xc002739ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc001f7b780?, {0x624a8a0?, 0xc001f7b740?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc002739af8 sp=0xc002739ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc005c201b0?}, {0xc0054afc68, 0x2, 0xc0054afdc0?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc002739b48 sp=0xc002739af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001766840, {0x624a948, 0xc005c201b0}, 0xc00327aac0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc002739e68 sp=0xc002739b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc0035bf0b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc002739f30 sp=0xc002739e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002739fe0 sp=0xc002739f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002739fe8 sp=0xc002739fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498444 [runnable]: runtime.gopark(0xae1b6449a8d?, 0x0?, 0x0?, 0x0?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ba51a48 sp=0xc00ba51a28 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00ba51ab0 sp=0xc00ba51a48 pc=0x4a17de sync.runtime_SemacquireMutex(0xc00ba51b70?, 0x20?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00ba51ae0 sp=0xc00ba51ab0 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x7f27517f93a0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00ba51b38 sp=0xc00ba51ae0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc001a1ab00?, 0xc00ba51bc8?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00ba51b80 sp=0xc00ba51b38 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc001a1ab00, {0x12?, 0x12?}, 0x0, 0xc012e7fd78, 0xc004c17c10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00ba51c80 sp=0xc00ba51b80 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc012e7fb00, 0x40?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00ba51d20 sp=0xc00ba51c80 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc012e7fb00, {0xc00809b840, 0x8, 0x1e}, 0xc00809b860?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc00ba51dd0 sp=0xc00ba51d20 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc012e7fb00?, {0xc00809b860?, 0x8?, 0x1e?}, {0xc00809b840?, 0x8?, 0x1e?}, 0x50?, 0x85?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc00ba51e38 sp=0xc00ba51dd0 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc004c18400?, {0xc00809b860?, 0x1?, 0x1e?}, {0xc00809b840?, 0x8?, 0x1e?}, 0x38?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00ba51e88 sp=0xc00ba51e38 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc004c18400, {0xc00809b860, 0x8, 0x1e}, {0xc00809b840, 0x8, 0x1e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00ba51ed8 sp=0xc00ba51e88 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc004c17e20, {0xc00809b840?, 0x0?, 0x81?}, 0x7?, 0x7?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00ba52008 sp=0xc00ba51ed8 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc004c17e20, {0xc00809b860?, 0xa?, 0x8?}, {0xc00809b840?, 0xc007136028?, 0xc00af9ba40?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00ba52048 sp=0xc00ba52008 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc004c17c88, {0xc00809b860?, 0x1e?, 0xc00809b840?}, {0xc00809b840, 0x8, 0x1e}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00ba520a0 sp=0xc00ba52048 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc004c17900, {0xc00809b840, 0x8, 0x1e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00ba52138 sp=0xc00ba520a0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc007728028, {{0xc0038ead68, 0x7, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00ba52188 sp=0xc00ba52138 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc010edd700, {{0xc0038ead68, 0x7, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:489 +0x14a fp=0xc00ba52248 sp=0xc00ba52188 pc=0x180cdaa github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x62b4f18, 0xc010edd700}, {{0xc0038ead68, 0x7, 0x8}, {0x0, 0x0, 0x0}}, 0xc0150d0c60) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1129 +0xc3 fp=0xc00ba52698 sp=0xc00ba52248 pc=0x1817843 github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x624a948, 0xc015350fc0}, {0x62b8438?, 0xc007728000?}, {0x62b4f18, 0xc010edd700}, 0xc00c684460, {0xc0038ead68, 0x7, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1650 +0x165 fp=0xc00ba533a8 sp=0xc00ba52698 pc=0x181aa25 github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x624a948, 0xc015350fc0}, {0x62b8438, 0xc007728000}, {0x62b4f18, 0xc010edd700}, 0x3?, {0xc0038ead68, 0x7, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1414 +0x1cd fp=0xc00ba534b0 sp=0xc00ba533a8 pc=0x18197cd github.com/cockroachdb/cockroach/pkg/storage.MVCCPut({0x624a948, 0xc015350fc0}, {0x7f275f2b7158?, 0xc007728000}, 0xc00c684460, {0xc0038ead68, 0x7, 0x8}, {0x17600a76dfa9ffd9, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1333 +0x2a5 fp=0xc00ba535c0 sp=0xc00ba534b0 pc=0x1819005 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Put({_, _}, {_, _}, {{0x62cc4b0, 0xc010e24480}, {{0x17600a76dfa9ffd9, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_put.go:59 +0x311 fp=0xc00ba53bf0 sp=0xc00ba535c0 pc=0x1cfa251 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76dfa9ffd9, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc00ba54480 sp=0xc00ba53bf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00dcecb40, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00ba562f0 sp=0xc00ba54480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc019553540, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00ba56940 sp=0xc00ba562f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc019553540, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00ba56e18 sp=0xc00ba56940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76fd7764d9, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00ba57c98 sp=0xc00ba56e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000580000?, {0x624a948, 0xc015350fc0}, {0xc0038ead80, 0x8}, 0xc00dcecb40, 0x8?, 0x0?, {{0x17600a76fd7764d9, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00ba58918 sp=0xc00ba57c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc015350fc0}, {0xc0038ead80, 0x8}, 0xc00dcecb40, 0x400000004?, 0xc00ba59968, {{0x17600a76fd7764d9, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00ba589d0 sp=0xc00ba58918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc001bc7000, {0x624a948, 0xc015350fc0}, 0xc00dcecb40, 0xc019553540, 0xc00ba59968, {{0x17600a76fd7764d9, 0x0, 0x0}, {0x17600a76e06082bf, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00ba59398 sp=0xc00ba589d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc001bc7000, {0x624a948, 0xc015350fc0}, 0xc00dcecb40, 0xc019553540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00ba59b08 sp=0xc00ba59398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001bc7000, {0x624a948, 0xc015350fc0}, 0xc00dcecb40, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00ba59e28 sp=0xc00ba59b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc001bc7000, {_, _}, {{{0x17600a76dfa9ffd9, 0x0, 0x0}, 0x0, {0x17600a76e3711ad8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00ba5a1f0 sp=0xc00ba59e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76dfa9ffd9, 0x0, 0x0}, 0x0, {0x17600a76e3711ad8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00ba5ab78 sp=0xc00ba5a1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xcc2934?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3711ad8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00ba5adc0 sp=0xc00ba5ab78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc015350f00}, {0xc01372d888?}, 0xc00dceca50) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00ba5b430 sp=0xc00ba5adc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc015350d80}, 0xc00dceca50) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00ba5b4d8 sp=0xc00ba5b430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc015350d80}, {0x52f0080?, 0xc00dceca50}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00ba5b518 sp=0xc00ba5b4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc015350d80}, {0x52f0080, 0xc00dceca50}, 0xc010c71820, 0xc013dcded8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00ba5b848 sp=0xc00ba5b518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc015350d80?}, {0x52f0080?, 0xc00dceca50?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00ba5b888 sp=0xc00ba5b848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc015350d80}, {0x52f0080, 0xc00dceca50}, 0xc00ba5b920?, 0xc00dad2340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00ba5b8b8 sp=0xc00ba5b888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc015350d80?}, {0x52f0080?, 0xc00dceca50?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00ba5b8f8 sp=0xc00ba5b8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc015350d80?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00ba5b930 sp=0xc00ba5b8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc015350d80}, {0x203004?, 0x40?}, 0xc00ba5b9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00ba5b9a8 sp=0xc00ba5b930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc015350d80?}, {0x52f0080?, 0xc00dceca50?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00ba5ba20 sp=0xc00ba5b9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc015350d80?}, {0x52f0080?, 0xc00dceca50?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00ba5ba60 sp=0xc00ba5ba20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc015350d80}, {0x52f0080, 0xc00dceca50}, 0xc010c71820, 0xc013dcded8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00ba5bab0 sp=0xc00ba5ba60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc015350d80}, 0xc00010f4a0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00ba5bb08 sp=0xc00ba5bab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc047d0d0e0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00ba5be48 sp=0xc00ba5bb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc047d0d0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00ba5bf68 sp=0xc00ba5be48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00ba5bfe0 sp=0xc00ba5bf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ba5bfe8 sp=0xc00ba5bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 1113 [select]: runtime.gopark(0xc003fadc60?, 0xa?, 0xa0?, 0xa8?, 0xc003fad8d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003fad6e8 sp=0xc003fad6c8 pc=0x4907d6 runtime.selectgo(0xc003fadc60, 0xc003fad8c0, 0xc0086dc1a0?, 0x0, 0x16?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003fad828 sp=0xc003fad6e8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0047ab980, {0x624a948?, 0xc0048174a0?}, 0x97?, 0xc00481a3c0, 0xc002a40360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:268 +0x51f fp=0xc003fadef0 sp=0xc003fad828 pc=0x1c85b3f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x624a948?, 0xc0048174a0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:221 +0x3c fp=0xc003fadf30 sp=0xc003fadef0 pc=0x1c855dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003fadfe0 sp=0xc003fadf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003fadfe8 sp=0xc003fadfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 385960 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x2c2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00aca0f58 sp=0xc00aca0f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbd8e0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00aca0f90 sp=0xc00aca0f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd7f8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00aca0fb0 sp=0xc00aca0f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00d03d300?, 0xc00d0c1000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00aca0fd8 sp=0xc00aca0fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00d03d300, {0xc00d0c1000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00aca1058 sp=0xc00aca0fd8 pc=0x535efa net.(*netFD).Read(0xc00d03d300, {0xc00d0c1000?, 0x7b5a68?, 0xc00d03d300?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00aca10a0 sp=0xc00aca1058 pc=0x7a20c9 net.(*conn).Read(0xc0061edb90, {0xc00d0c1000?, 0x164290f?, 0xc0061edb90?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00aca10e8 sp=0xc00aca10a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01234acc0, {0xc00d0c1000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00aca1130 sp=0xc00aca10e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03b44b2a8, {0xc00d0c1000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00aca11d0 sp=0xc00aca1130 pc=0x370f896 bufio.(*Reader).fill(0xc0077e9090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00aca1208 sp=0xc00aca11d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0077e9090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00aca1228 sp=0xc00aca1208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0077e9668?, {0x62295c8, 0xc0077e9090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00aca1260 sp=0xc00aca1228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0077e9000, 0xc0474990e0, 0xc00aca1442, 0xc00aca1444, {0x623b718, 0xc00421fb00}, 0xc0059fa000, 0xc00a6925b4, 0xc00aca1440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00aca1398 sp=0xc00aca1260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0077e9000, {0x624a948, 0xc0222c9860}, 0xc0474990d0, 0xc0059fa000, 0xc0222c97a0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00aca1768 sp=0xc00aca1398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0222c9860}, {0x628a538, 0xc01234acc0}, {{{0xc003c62038, 0x4}}, 0x1, 0xc0222c97d0, 0xc0222c9800, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00aca18a8 sp=0xc00aca1768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0222c9860}, {0x628a538?, 0xc01234acc0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00aca1e20 sp=0xc00aca18a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0222c8fc0?}, {0x628a538, 0xc01234acc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00aca1eb0 sp=0xc00aca1e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0222c8fc0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00aca1f30 sp=0xc00aca1eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00aca1fe0 sp=0xc00aca1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00aca1fe8 sp=0xc00aca1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1165 [select]: runtime.gopark(0xc0032deed8?, 0x2?, 0xa0?, 0xbb?, 0xc0032dee54?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032deca0 sp=0xc0032dec80 pc=0x4907d6 runtime.selectgo(0xc0032deed8, 0xc0032dee50, 0x217?, 0x0, 0xfb4477?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032dede0 sp=0xc0032deca0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).Start.func1({0xc00448e000, 0xc0032a1e90}) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:348 +0x255 fp=0xc0032def30 sp=0xc0032dede0 pc=0x25c44f5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0032defe0 sp=0xc0032def30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032defe8 sp=0xc0032defe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1166 [select, 9 minutes]: runtime.gopark(0xc003dd1ee0?, 0x4?, 0xf6?, 0xda?, 0xc003dd1e98?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006645cf8 sp=0xc006645cd8 pc=0x4907d6 runtime.selectgo(0xc006645ee0, 0xc003dd1e90, 0x1?, 0x0, 0xc00449b950?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006645e38 sp=0xc006645cf8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1({0x624a948, 0xc00449b950}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/provider.go:137 +0x24d fp=0xc006645f30 sp=0xc006645e38 pc=0x26a290d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006645fe0 sp=0xc006645f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006645fe8 sp=0xc006645fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1167 [select]: runtime.gopark(0xc006643ef0?, 0x2?, 0x68?, 0x0?, 0xc006643e4c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006643c90 sp=0xc006643c70 pc=0x4907d6 runtime.selectgo(0xc006643ef0, 0xc006643e48, 0x0?, 0x0, 0x7ef3537643?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006643dd0 sp=0xc006643c90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*jobMonitor).start.func1({0x624a948, 0xc00449b980}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/scheduled_job_monitor.go:96 +0x2a7 fp=0xc006643f30 sp=0xc006643dd0 pc=0x26a31a7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006643fe0 sp=0xc006643f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006643fe8 sp=0xc006643fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1168 [select, 9 minutes]: runtime.gopark(0xc00555fef8?, 0x2?, 0x6b?, 0xb4?, 0xc00555febc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006647d30 sp=0xc006647d10 pc=0x4907d6 runtime.selectgo(0xc006647ef8, 0xc00555feb8, 0x624a8a0?, 0x0, 0x2c?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006647e70 sp=0xc006647d30 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller.(*Controller).Start.func2({0x624a948?, 0xc00449b9b0?}) github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/controller.go:122 +0x13c fp=0xc006647f30 sp=0xc006647e70 pc=0x26a8f3c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006647fe0 sp=0xc006647f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006647fe8 sp=0xc006647fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 488462 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc015d69640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc015d695d0 sp=0xc015d695b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00a56ff90, 0x2c) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc015d69618 sp=0xc015d695d0 pc=0x4be30c sync.(*Cond).Wait(0xc00328c800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc015d69650 sp=0xc015d69618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00328ce68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc015d69708 sp=0xc015d69650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00404f900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc015d69cb8 sp=0xc015d69708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00404f900, {0x624a8a0, 0xc00a56ffc0}, 0x4119c41c00000000?, 0xc0064f5dc0?, 0xc00d91a1a0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc015d69d70 sp=0xc015d69cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00a56ffc0?}, {0xc0059fa000?}, 0x3?, 0xc00a1983d8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc015d69e20 sp=0xc015d69d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc015d69fe0 sp=0xc015d69e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc015d69fe8 sp=0xc015d69fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 1186 [select, 9 minutes]: runtime.gopark(0xc0032ecf00?, 0x2?, 0xf6?, 0xda?, 0xc0032ece9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005fe7cf0 sp=0xc005fe7cd0 pc=0x4907d6 runtime.selectgo(0xc005fe7f00, 0xc0032ece98, 0x0?, 0x0, 0x12ecc7396?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005fe7e30 sp=0xc005fe7cf0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).Start.func1({0x624a948, 0xc00449ba40}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:84 +0x27a fp=0xc005fe7f30 sp=0xc005fe7e30 pc=0x269563a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005fe7fe0 sp=0xc005fe7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005fe7fe8 sp=0xc005fe7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1187 [select]: runtime.gopark(0xc00184ef00?, 0x2?, 0x9?, 0x0?, 0xc00184eedc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00184ed68 sp=0xc00184ed48 pc=0x4907d6 runtime.selectgo(0xc00184ef00, 0xc00184eed8, 0xc0100de000?, 0x0, 0xc00419fa40?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00184eea8 sp=0xc00184ed68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func1({0xc0041d6380?, 0xc00419fa40?}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:70 +0xcb fp=0xc00184ef30 sp=0xc00184eea8 pc=0x1f89feb github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00184efe0 sp=0xc00184ef30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00184efe8 sp=0xc00184efe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1188 [select]: runtime.gopark(0xc003634f00?, 0x2?, 0x80?, 0x1?, 0xc003634ee4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003634d70 sp=0xc003634d50 pc=0x4907d6 runtime.selectgo(0xc003634f00, 0xc003634ee0, 0xc4487c?, 0x0, 0x6295c98?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003634eb0 sp=0xc003634d70 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func2({0xc0041d6700?, 0xc00419f860?}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:87 +0x96 fp=0xc003634f30 sp=0xc003634eb0 pc=0x1f89ef6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003634fe0 sp=0xc003634f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003634fe8 sp=0xc003634fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1189 [select]: runtime.gopark(0xc001127f00?, 0x2?, 0xa0?, 0x7d?, 0xc001127ee4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001127d70 sp=0xc001127d50 pc=0x4907d6 runtime.selectgo(0xc001127f00, 0xc001127ee0, 0xc4487c?, 0x0, 0x6295c98?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001127eb0 sp=0xc001127d70 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache.(*Cache).Start.func1({0xc0041d6a80?, 0xc00419f560?}) github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache/txn_id_cache.go:145 +0xa5 fp=0xc001127f30 sp=0xc001127eb0 pc=0x26abaa5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001127fe0 sp=0xc001127f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001127fe8 sp=0xc001127fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1190 [select]: runtime.gopark(0xc00dff5e68?, 0x5?, 0x50?, 0x56?, 0xc00dff5d8e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dff5bd8 sp=0xc00dff5bb8 pc=0x4907d6 runtime.selectgo(0xc00dff5e68, 0xc00dff5d84, 0xc0000000af?, 0x0, 0x22?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dff5d18 sp=0xc00dff5bd8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1({0x624a948, 0xc00449bb00}) github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:394 +0x1dc fp=0xc00dff5f30 sp=0xc00dff5d18 pc=0x1aebcfc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00dff5fe0 sp=0xc00dff5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dff5fe8 sp=0xc00dff5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1191 [select]: runtime.gopark(0xc00cecdec8?, 0x3?, 0x0?, 0x0?, 0xc00cecde1a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cecdca0 sp=0xc00cecdc80 pc=0x4907d6 runtime.selectgo(0xc00cecdec8, 0xc00cecde14, 0xc007245430?, 0x0, 0xfb4477?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00cecdde0 sp=0xc00cecdca0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc005922960, {0x624a948, 0xc00449bb60}) github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:199 +0x245 fp=0xc00cecdf08 sp=0xc00cecdde0 pc=0x26bdc05 github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll-fm({0x624a948?, 0xc00449bb60?}) :1 +0x39 fp=0xc00cecdf30 sp=0xc00cecdf08 pc=0x26c2319 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cecdfe0 sp=0xc00cecdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cecdfe8 sp=0xc00cecdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1192 [semacquire, 9 minutes]: runtime.gopark(0xc003eb1a68?, 0x4615d9?, 0x0?, 0x6?, 0xc004766900?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003eb19f0 sp=0xc003eb19d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc004771690, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc003eb1a58 sp=0xc003eb19f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc003eb1a88 sp=0xc003eb1a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc003eb1ae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc003eb1ab0 sp=0xc003eb1a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc004771680) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc003eb1ac8 sp=0xc003eb1ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004771680?, {0x624a8a0?, 0xc004771640?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc003eb1af8 sp=0xc003eb1ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc0048bafc0?}, {0xc003eb1c68, 0x2, 0x4000000000000000?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc003eb1b48 sp=0xc003eb1af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001734580, {0x624a948, 0xc0048bafc0}, 0xc0045ff000) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc003eb1e68 sp=0xc003eb1b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc00449bc50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc003eb1f30 sp=0xc003eb1e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003eb1fe0 sp=0xc003eb1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003eb1fe8 sp=0xc003eb1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1193 [semacquire, 9 minutes]: runtime.gopark(0xc003eb3a68?, 0x4615d9?, 0x20?, 0x7?, 0xc004766990?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003eb39f0 sp=0xc003eb39d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc004771710, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc003eb3a58 sp=0xc003eb39f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc003eb3a88 sp=0xc003eb3a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc003eb3ae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc003eb3ab0 sp=0xc003eb3a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc004771700) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc003eb3ac8 sp=0xc003eb3ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004771700?, {0x624a8a0?, 0xc0047716c0?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc003eb3af8 sp=0xc003eb3ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc0048bb110?}, {0xc003eb3c68, 0x2, 0x4000000000000000?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc003eb3b48 sp=0xc003eb3af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0017346e0, {0x624a948, 0xc0048bb110}, 0xc0045ff100) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc003eb3e68 sp=0xc003eb3b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc00449bd40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc003eb3f30 sp=0xc003eb3e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003eb3fe0 sp=0xc003eb3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003eb3fe8 sp=0xc003eb3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1194 [select, 3 minutes]: runtime.gopark(0xc00b355ea0?, 0x2?, 0x60?, 0x5c?, 0xc00b355e1c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0106cdc60 sp=0xc0106cdc40 pc=0x4907d6 runtime.selectgo(0xc0106cdea0, 0xc00b355e18, 0x3?, 0x0, 0xc00449bda0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0106cdda0 sp=0xc0106cdc60 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1({0x624a948, 0xc00449bda0}) github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1079 +0x105 fp=0xc0106cdf30 sp=0xc0106cdda0 pc=0x1ab54e5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0106cdfe0 sp=0xc0106cdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0106cdfe8 sp=0xc0106cdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1195 [select, 1 minutes]: runtime.gopark(0xc005607e08?, 0x2?, 0x0?, 0x0?, 0xc005607ddc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0053edc18 sp=0xc0053edbf8 pc=0x4907d6 runtime.selectgo(0xc0053ede08, 0xc005607dd8, 0xc00449bdd0?, 0x0, 0xc000aa45a0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0053edd58 sp=0xc0053edc18 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1({0x624a948, 0xc00449bdd0}) github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1185 +0x2d3 fp=0xc0053edf30 sp=0xc0053edd58 pc=0x1ab6493 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0053edfe0 sp=0xc0053edf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0053edfe8 sp=0xc0053edfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1196 [select]: runtime.gopark(0xc012e63ee0?, 0x3?, 0x18?, 0x3d?, 0xc012e63e5a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012e63cc0 sp=0xc012e63ca0 pc=0x4907d6 runtime.selectgo(0xc012e63ee0, 0xc012e63e54, 0x0?, 0x0, 0x3bf4868e8d89b4f6?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc012e63e00 sp=0xc012e63cc0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8({0x624a948?, 0xc00449be90?}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:822 +0x196 fp=0xc012e63f30 sp=0xc012e63e00 pc=0x229c396 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012e63fe0 sp=0xc012e63f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012e63fe8 sp=0xc012e63fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1197 [select, 9 minutes]: runtime.gopark(0xc005b08ee0?, 0x3?, 0x40?, 0x0?, 0xc005b08e32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b08c98 sp=0xc005b08c78 pc=0x4907d6 runtime.selectgo(0xc005b08ee0, 0xc005b08e2c, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005b08dd8 sp=0xc005b08c98 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9({0x624a948?, 0xc00449bef0?}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:854 +0x18d fp=0xc005b08f30 sp=0xc005b08dd8 pc=0x229c02d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005b08fe0 sp=0xc005b08f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b08fe8 sp=0xc005b08fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1198 [select]: runtime.gopark(0xc00b7d2de0?, 0x3?, 0xc0?, 0x82?, 0xc00b7d2dda?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b7d2c60 sp=0xc00b7d2c40 pc=0x4907d6 runtime.selectgo(0xc00b7d2de0, 0xc00b7d2dd4, 0x1000101010000?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b7d2da0 sp=0xc00b7d2c60 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).firstResult(0xc005e4f640, {0x624a948, 0xc008365050}) github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:116 +0xf9 fp=0xc00b7d2ef8 sp=0xc00b7d2da0 pc=0x3429bb9 github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).nextResult(0xc005e4f640?, {0x624a948, 0xc008365050}) github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:155 +0xfe fp=0xc00b7d3018 sp=0xc00b7d2ef8 pc=0x342a23e github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).execInternal(0xc000aa45a0, {0x624a8a0, 0xc004771840}, {0x5356b51, 0xa}, 0xc005e4f640, 0x6223810?, {{{0x53343c6, 0x4}}, {0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/internal.go:1015 +0x1285 fp=0xc00b7d3598 sp=0xc00b7d3018 pc=0x3426765 github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).ExecEx(0x46159f?, {0x624a8a0, 0xc004771840}, {0x5356b51, 0xa}, 0x6207480?, {{{0x53343c6, 0x4}}, {0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/internal.go:688 +0x19b fp=0xc00b7d3678 sp=0xc00b7d3598 pc=0x3424b1b github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).Exec(0x0?, {0x624a8a0, 0xc004771840}, {0x5356b51, 0xa}, 0x0?, {0x5558288, 0x108}, {0xc008364fc0, 0x3, ...}) github.com/cockroachdb/cockroach/pkg/sql/internal.go:669 +0x165 fp=0xc00b7d3780 sp=0xc00b7d3678 pc=0x3424905 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).claimJobs.func1({0x624a8a0, 0xc004771840}, 0x62924c0?) github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:100 +0x27a fp=0xc00b7d3860 sp=0xc00b7d3780 pc=0x228539a github.com/cockroachdb/cockroach/pkg/kv.runTxn.func1({0x624a8a0?, 0xc004771840?}, 0xc016883ad0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:980 +0x27 fp=0xc00b7d3888 sp=0xc00b7d3860 pc=0x16ae427 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0xc00ec99080, {0x624a8a0, 0xc004771840}, 0xc00b7d3bc8) github.com/cockroachdb/cockroach/pkg/kv/txn.go:961 +0xae fp=0xc00b7d3b70 sp=0xc00b7d3888 pc=0x16b6d0e github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x624a8a0, 0xc004771840}, 0x7?, 0x8364f90?) github.com/cockroachdb/cockroach/pkg/kv/db.go:979 +0x5a fp=0xc00b7d3be8 sp=0xc00b7d3b70 pc=0x16ae33a github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x3936480?, {0x624a8a0, 0xc004771840}, 0x62146e0?, 0x0?, 0xc008364f90?) github.com/cockroachdb/cockroach/pkg/kv/db.go:942 +0x89 fp=0xc00b7d3c30 sp=0xc00b7d3be8 pc=0x16ae0c9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:921 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).claimJobs(0xc00a1b5600?, {0x624a8a0?, 0xc004771840?}, {0x6267438?, 0xc004738a80?}) github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:94 +0x65 fp=0xc00b7d3c88 sp=0xc00b7d3c30 pc=0x22850c5 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func5({0x624a8a0, 0xc004771840}, {0x6267438, 0xc004738a80}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:773 +0xa5 fp=0xc00b7d3d00 sp=0xc00b7d3c88 pc=0x229cbe5 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).withSession(0xc00a1b5600, {0x624a8a0, 0xc004771840}, 0xc006889dd0) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:711 +0x26a fp=0xc00b7d3dc8 sp=0xc00b7d3d00 pc=0x229b4aa github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func12({0x624a8a0?, 0xc004771840?}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:727 +0x34 fp=0xc00b7d3df8 sp=0xc00b7d3dc8 pc=0x229cb14 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10({0x624a948?, 0xc00449bf80?}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:890 +0x237 fp=0xc00b7d3f30 sp=0xc00b7d3df8 pc=0x229bdf7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b7d3fe0 sp=0xc00b7d3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b7d3fe8 sp=0xc00b7d3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1199 [select, 9 minutes]: runtime.gopark(0xc00be15eb0?, 0x4?, 0x86?, 0xfc?, 0xc00be15e38?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c24acb8 sp=0xc00c24ac98 pc=0x4907d6 runtime.selectgo(0xc00c24aeb0, 0xc00be15e30, 0xc006889eb0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c24adf8 sp=0xc00c24acb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).run(0xc005922a00, {0x624a948, 0xc004728000}) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:160 +0x3b3 fp=0xc00c24af08 sp=0xc00c24adf8 pc=0x391cc33 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).Start.func1({0x624a948?, 0xc004728000?}) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:102 +0x30 fp=0xc00c24af30 sp=0xc00c24af08 pc=0x391c850 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c24afe0 sp=0xc00c24af30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c24afe8 sp=0xc00c24afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3757 [chan receive]: runtime.gopark(0xc0085bddd0?, 0x105495b?, 0x48?, 0xa9?, 0xc0028e7b30?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a61cf8 sp=0xc005a61cd8 pc=0x4907d6 runtime.chanrecv(0xc0079fe9c0, 0xc0085bddc0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc005a61d88 sp=0xc005a61cf8 pc=0x45a91b runtime.chanrecv2(0x4618e7?, 0x28?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc005a61db0 sp=0xc005a61d88 pc=0x45a458 github.com/cockroachdb/cmux.muxListener.Accept(...) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 github.com/cockroachdb/cmux.(*muxListener).Accept(0xc002a40360?) :1 +0x45 fp=0xc005a61de0 sp=0xc005a61db0 pc=0x1642445 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc0005499a0, {0x624a948, 0xc0028e7b30}, 0x6295c98?, {0x6248348, 0xc001591800}, 0xc002c9bc08) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:138 +0x82 fp=0xc005a61eb8 sp=0xc005a61de0 pc=0x1643602 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1({0x624a948, 0xc0028e7b30}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1470 +0xd3 fp=0xc005a61f30 sp=0xc005a61eb8 pc=0x39e1bb3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005a61fe0 sp=0xc005a61f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a61fe8 sp=0xc005a61fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3793 [runnable]: runtime.Gosched(...) GOROOT/src/runtime/proc.go:318 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0040e24e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:570 +0x1f6 fp=0xc01087df88 sp=0xc01087df30 pc=0xbd7156 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc01087dfe0 sp=0xc01087df88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01087dfe8 sp=0xc01087dfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 1112 [select, 9 minutes]: runtime.gopark(0xc0032ddfb0?, 0x2?, 0x58?, 0xde?, 0xc0032ddf9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032dde28 sp=0xc0032dde08 pc=0x4907d6 runtime.selectgo(0xc0032ddfb0, 0xc0032ddf98, 0xc002a40360?, 0x0, 0xc00694dd30?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032ddf68 sp=0xc0032dde28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0032ddfe0 sp=0xc0032ddf68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032ddfe8 sp=0xc0032ddfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 3723 [semacquire, 9 minutes]: runtime.gopark(0x18?, 0xc00072d400?, 0x80?, 0x93?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002f4faa8 sp=0xc002f4fa88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc001f7bb50, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc002f4fb10 sp=0xc002f4faa8 pc=0x4a17de sync.runtime_Semacquire(0xc001f7bb40?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc002f4fb40 sp=0xc002f4fb10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc002f4fb68 sp=0xc002f4fb40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc001f7bb40) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc002f4fb80 sp=0xc002f4fb68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc001f7bb40?, {0x624a8a0?, 0xc001f7bb00?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc002f4fbb0 sp=0xc002f4fb80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc001f7b740}, {0xc001f7bac0, 0x1, 0xc811b3403ec62b62?}, 0x0, 0xc006e07aa0, {0xc00833be40, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc002f4fd38 sp=0xc002f4fbb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc006979d8e?, {0x624a8a0, 0xc001f7b740}, {0xc0035bf020, 0x1, 0xc448e6?}, {0x0?, 0x6979f00?, 0xc0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc002f4fe58 sp=0xc002f4fd38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc006979ee8?, {0x624a8a0?, 0xc001f7b740?}, {0xc0035bf020?, 0xc006979d30?, 0xc006979fd0?}, {0x1054c46?, 0x624a948?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc002f4fed0 sp=0xc002f4fe58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc001f7b740?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc002f4ff58 sp=0xc002f4fed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc002f4ff78 sp=0xc002f4ff58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc002f4ffe0 sp=0xc002f4ff78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002f4ffe8 sp=0xc002f4ffe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498452 [select]: runtime.gopark(0xc00b363808?, 0x3?, 0x4?, 0x0?, 0xc00b3634d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b363258 sp=0xc00b363238 pc=0x4907d6 runtime.selectgo(0xc00b363808, 0xc00b3634cc, 0x17600a76a63819fc?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b363398 sp=0xc00b363258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc006b2e390}, 0xc0081da3c0, 0xc0195534a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00b363b08 sp=0xc00b363398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc006b2e390}, 0xc0081da3c0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00b363e28 sp=0xc00b363b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76a5521f84, 0x0, 0x0}, 0x0, {0x17600a76e197d9ec, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00b3641f0 sp=0xc00b363e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76a5521f84, 0x0, 0x0}, 0x0, {0x17600a76e197d9ec, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00b364b78 sp=0xc00b3641f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e197d9ec, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00b364dc0 sp=0xc00b364b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00796fef0}, {0xc00b365470?}, 0xc0081da1e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00b365430 sp=0xc00b364dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00796fa70}, 0xc0081da1e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00b3654d8 sp=0xc00b365430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc00796fa70}, {0x52f0080?, 0xc0081da1e0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00b365518 sp=0xc00b3654d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00796fa70}, {0x52f0080, 0xc0081da1e0}, 0xc011dbb920, 0xc001c33fc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00b365848 sp=0xc00b365518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00796fa70?}, {0x52f0080?, 0xc0081da1e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00b365888 sp=0xc00b365848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00796fa70}, {0x52f0080, 0xc0081da1e0}, 0xc00b365920?, 0xc00a1d7d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00b3658b8 sp=0xc00b365888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00796fa70?}, {0x52f0080?, 0xc0081da1e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b3658f8 sp=0xc00b3658b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00796fa70?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00b365930 sp=0xc00b3658f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00796fa70}, {0x203004?, 0x40?}, 0xc00b3659e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00b3659a8 sp=0xc00b365930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00796fa70?}, {0x52f0080?, 0xc0081da1e0?}, 0x5003b00?, 0x7f278cfaaa01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00b365a20 sp=0xc00b3659a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00796fa70?}, {0x52f0080?, 0xc0081da1e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b365a60 sp=0xc00b365a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc00796fa70}, {0x52f0080, 0xc0081da1e0}, 0xc011dbb920, 0xc001c33fc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00b365ab0 sp=0xc00b365a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc00796fa70}, 0xc00823a180, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00b365b08 sp=0xc00b365ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc010611560, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00b365e48 sp=0xc00b365b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc010611560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00b365f68 sp=0xc00b365e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00b365fe0 sp=0xc00b365f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b365fe8 sp=0xc00b365fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 1206 [semacquire, 9 minutes]: runtime.gopark(0x18?, 0xc00072d400?, 0xc0?, 0xf?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005ae9aa8 sp=0xc005ae9a88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc004771990, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc005ae9b10 sp=0xc005ae9aa8 pc=0x4a17de sync.runtime_Semacquire(0xc004771980?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc005ae9b40 sp=0xc005ae9b10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc005ae9b68 sp=0xc005ae9b40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc004771980) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc005ae9b80 sp=0xc005ae9b68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004771980?, {0x624a8a0?, 0xc004771940?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc005ae9bb0 sp=0xc005ae9b80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc004771640}, {0xc004771900, 0x1, 0xfc4d45?}, 0x0, 0xc0031bf860, {0xc007245980, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc005ae9d38 sp=0xc005ae9bb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc0045ff000?, {0x624a8a0, 0xc004771640}, {0xc00449bbf0, 0x1, 0x100000000c448e6?}, {0x1d747cf?, 0x2?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc005ae9e58 sp=0xc005ae9d38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x624a948?, {0x624a8a0?, 0xc004771640?}, {0xc00449bbf0?, 0xc0032b7700?, 0xc0032b77d0?}, {0x1054c46?, 0x624a948?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc005ae9ed0 sp=0xc005ae9e58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc004771640?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc005ae9f58 sp=0xc005ae9ed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc005ae9f78 sp=0xc005ae9f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc005ae9fe0 sp=0xc005ae9f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005ae9fe8 sp=0xc005ae9fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1207 [select]: runtime.gopark(0xc003eb2e18?, 0x2?, 0xb0?, 0x78?, 0xc003eb2dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003eb2c68 sp=0xc003eb2c48 pc=0x4907d6 runtime.selectgo(0xc003eb2e18, 0xc003eb2df8, 0xc0045ff000?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003eb2da8 sp=0xc003eb2c68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001734580, {0x624a8a0, 0xc004771640}, 0xc0059b8fd0?, 0xc0031bf860) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc003eb2f20 sp=0xc003eb2da8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc004771640?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc003eb2f58 sp=0xc003eb2f20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc003eb2f78 sp=0xc003eb2f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc003eb2fe0 sp=0xc003eb2f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003eb2fe8 sp=0xc003eb2fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1208 [semacquire, 9 minutes]: runtime.gopark(0x18?, 0xc00072d400?, 0xa0?, 0x34?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005ae5aa8 sp=0xc005ae5a88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc004771bd0, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc005ae5b10 sp=0xc005ae5aa8 pc=0x4a17de sync.runtime_Semacquire(0xc004771bc0?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc005ae5b40 sp=0xc005ae5b10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc005ae5b68 sp=0xc005ae5b40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc004771bc0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc005ae5b80 sp=0xc005ae5b68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004771bc0?, {0x624a8a0?, 0xc004771b80?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc005ae5bb0 sp=0xc005ae5b80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc0047716c0}, {0xc004771b40, 0x1, 0xfc4d9b?}, 0x0, 0xc0031bf980, {0xc0072459d0, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc005ae5d38 sp=0xc005ae5bb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00419ede0?, {0x624a8a0, 0xc0047716c0}, {0xc00449bce0, 0x1, 0x100000000c448e6?}, {0x1d747cf?, 0x2?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc005ae5e58 sp=0xc005ae5d38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x533b86e?, {0x624a8a0?, 0xc0047716c0?}, {0xc00449bce0?, 0xc0032b7f00?, 0xc0032b7fd0?}, {0x1054c46?, 0x624a948?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc005ae5ed0 sp=0xc005ae5e58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc0047716c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc005ae5f58 sp=0xc005ae5ed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc005ae5f78 sp=0xc005ae5f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc005ae5fe0 sp=0xc005ae5f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005ae5fe8 sp=0xc005ae5fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1209 [select]: runtime.gopark(0xc004d36e18?, 0x2?, 0x18?, 0xb7?, 0xc004d36dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004d36c68 sp=0xc004d36c48 pc=0x4907d6 runtime.selectgo(0xc004d36e18, 0xc004d36df8, 0xc0045ff100?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004d36da8 sp=0xc004d36c68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0017346e0, {0x624a8a0, 0xc0047716c0}, 0xc0059b8fd0?, 0xc0031bf980) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc004d36f20 sp=0xc004d36da8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc0047716c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc004d36f58 sp=0xc004d36f20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc004d36f78 sp=0xc004d36f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc004d36fe0 sp=0xc004d36f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004d36fe8 sp=0xc004d36fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1210 [select, 9 minutes]: runtime.gopark(0xc005b0ae58?, 0x2?, 0x1?, 0x30?, 0xc005b0adfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b0ac80 sp=0xc005b0ac60 pc=0x4907d6 runtime.selectgo(0xc005b0ae58, 0xc005b0adf8, 0xc004771940?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005b0adc0 sp=0xc005b0ac80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc004771940}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc005b0af58 sp=0xc005b0adc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc005b0af78 sp=0xc005b0af58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc005b0afe0 sp=0xc005b0af78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b0afe8 sp=0xc005b0afe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1110 [select]: runtime.gopark(0xc003dcd168?, 0x3?, 0xdf?, 0xf2?, 0xc003dcd102?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003dccf78 sp=0xc003dccf58 pc=0x4907d6 runtime.selectgo(0xc003dcd168, 0xc003dcd0fc, 0x0?, 0x0, 0xc003dcd130?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003dcd0b8 sp=0xc003dccf78 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc006d30630, {0x624a948, 0xc004817140}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1217 +0xe8 fp=0xc003dcd1a8 sp=0xc003dcd0b8 pc=0x165bee8 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc006d30630?, 0x5684220?}, {0x624a948?, 0xc004817140?}, {0x51d3160, 0xc0134d3380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1253 +0x45 fp=0xc003dcd1e0 sp=0xc003dcd1a8 pc=0x165c0c5 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1301 github.com/cockroachdb/cockroach/pkg/rpc.(*clientStream).RecvMsg(0x0?, {0x51d3160?, 0xc0134d3380?}) :1 +0x77 fp=0xc003dcd248 sp=0xc003dcd1e0 pc=0x1666f77 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0007c3c80, {0x51d3160?, 0xc0134d3380?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc003dcd290 sp=0xc003dcd248 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x627d528?, 0xc0007c3c80?}}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1028 +0x59 fp=0xc003dcd2d0 sp=0xc003dcd290 pc=0x165b0b9 github.com/cockroachdb/cockroach/pkg/rpc.(*rangeFeedClientAdapter).Recv(0xc003dcd300?) :1 +0x34 fp=0xc003dcd2f0 sp=0xc003dcd2d0 pc=0x1666a34 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc003dcd310 sp=0xc003dcd2f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00749f810, 0xc003dcd780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc003dcd3b8 sp=0xc003dcd310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc004771940?}, {{0xc0045eeb08, 0x1, 0x8}, {0xc0045eeb10, 0x1, 0x8}}, {0x176009f8256f1fe6, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc003dcd9a0 sp=0xc003dcd3b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc004771940}, 0xc0072cf770, 0x5683b00?, {{0xc0045eeb08, 0x1, 0x8}, {0xc0045eeb10, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc003dcde80 sp=0xc003dcd9a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc004771940?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc003dcdf58 sp=0xc003dcde80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc003dcdf78 sp=0xc003dcdf58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc003dcdfe0 sp=0xc003dcdf78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003dcdfe8 sp=0xc003dcdfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1111 [chan receive, 9 minutes]: runtime.gopark(0xc0059d1180?, 0x624a948?, 0xe0?, 0x84?, 0xc000920000?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002e0b558 sp=0xc002e0b538 pc=0x4907d6 runtime.chanrecv(0xc0043aef00, 0xc00439b6f8, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc002e0b5e8 sp=0xc002e0b558 pc=0x45a91b runtime.chanrecv1(0x176009f8256f1fe6?, 0x0?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc002e0b610 sp=0xc002e0b5e8 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc005950c00, 0x97?, 0xc0077c8d80, {0x7f275f2bd8e0, 0xc006d30770}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:250 +0x707 fp=0xc002e0b7f0 sp=0xc002e0b610 pc=0x1e00027 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:152 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00679ea80, 0xc0077c8d80, {0x7f275f2bd8e0, 0xc006d30770}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3253 +0x10c fp=0xc002e0b850 sp=0xc002e0b7f0 pc=0x1e3af0c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc0007c3d20?, 0xc0077c8d80, {0x7f275f2bd8e0, 0xc006d30770}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:228 +0xf6 fp=0xc002e0b8b8 sp=0xc002e0b850 pc=0x1e64156 github.com/cockroachdb/cockroach/pkg/server.(*Node).singleRangeFeed(0x45efeb?, 0x50c16c0?, {0x7f275f2bd8e0?, 0xc006d30770?}) github.com/cockroachdb/cockroach/pkg/server/node.go:1341 +0x38 fp=0xc002e0b910 sp=0xc002e0b8b8 pc=0x39c1118 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x50c16c0?, 0xc00488fd40?, {0x62893f0?, 0xc006d30770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1335 +0x4e fp=0xc002e0b950 sp=0xc002e0b910 pc=0x39c108e github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x624a948?}, {0x627d570?, 0xc0007c3d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:980 +0x72 fp=0xc002e0b9a0 sp=0xc002e0b950 pc=0x165b012 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x52b4780, 0xc00a1cf500}, {0x6279ce8?, 0xc0048171d0?}, 0x8fb69e0, 0xc0043aeea0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc002e0bcd8 sp=0xc002e0b9a0 pc=0x16481a4 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x52b4780?, 0xc00a1cf500?}, {0x6279ce8?, 0xc0048171d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:827 +0x5b fp=0xc002e0bd18 sp=0xc002e0bcd8 pc=0x165a09b github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x52b4780, 0xc00a1cf500}, {0x6279ce8, 0xc0048171d0}, 0x1054f90?, 0xc0043b13c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc002e0bd48 sp=0xc002e0bd18 pc=0x16576d4 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x52b4780?, 0xc00a1cf500?}, {0x6279ce8?, 0xc0048171d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:830 +0x83 fp=0xc002e0bd88 sp=0xc002e0bd48 pc=0x165a0c3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x4c25e7?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc002e0bdb8 sp=0xc002e0bd88 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0048171a0}, {0xc0011e6ec0?, 0x4615d9?}, 0xc0011e6e78) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc002e0be30 sp=0xc002e0bdb8 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x52b4780, 0xc00a1cf500}, {0x6279ce8?, 0xc0048171d0?}, 0x8fb69e0, 0xc0043b13c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc002e0beb8 sp=0xc002e0be30 pc=0x165790f github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x52b4780?, 0xc00a1cf500?}, {0x6279ce8?, 0xc0048171d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:830 +0x83 fp=0xc002e0bef8 sp=0xc002e0beb8 pc=0x165a0c3 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc001a44c20, 0x3, 0x4}, {0x52b4780, 0xc00a1cf500}, {0x6279ce8, 0xc0048171d0}, 0x8fb69e0, 0xc0043aeea0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:832 +0x142 fp=0xc002e0bf30 sp=0xc002e0bef8 pc=0x1659fa2 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:985 +0x117 fp=0xc002e0bfe0 sp=0xc002e0bf30 pc=0x165af17 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002e0bfe8 sp=0xc002e0bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:973 +0x2dd goroutine 1213 [select, 9 minutes]: runtime.gopark(0xc0032b7e58?, 0x2?, 0x1?, 0x30?, 0xc0032b7dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b0bc80 sp=0xc005b0bc60 pc=0x4907d6 runtime.selectgo(0xc005b0be58, 0xc0032b7df8, 0xc004771b80?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005b0bdc0 sp=0xc005b0bc80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc004771b80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc005b0bf58 sp=0xc005b0bdc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc005b0bf78 sp=0xc005b0bf58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc005b0bfe0 sp=0xc005b0bf78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b0bfe8 sp=0xc005b0bfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1215 [select]: runtime.gopark(0xc008cf6d68?, 0x2?, 0x0?, 0x3a?, 0xc008cf6d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008cf6bb8 sp=0xc008cf6b98 pc=0x4907d6 runtime.selectgo(0xc008cf6d68, 0xc008cf6d30, 0x8?, 0x0, 0xc0231de000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008cf6cf8 sp=0xc008cf6bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0072cfef0, {0xc0048d0220, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc008cf6d98 sp=0xc008cf6cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0072cfef0, {0xc0048d0220?, 0xc00345dde8?, 0xc008cf6de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc008cf6dd8 sp=0xc008cf6d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0048bbd10, {0xc0048d0220?, 0xc008cf6e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc008cf6e20 sp=0xc008cf6dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc0048bbd10}, {0xc0048d0220, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc008cf6e68 sp=0xc008cf6e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc007e73200, {0xc0048d0220, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc008cf6ea8 sp=0xc008cf6e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc0048d0210, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc008cf6f28 sp=0xc008cf6ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc008117028?, 0xc007e73200, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc008cf6fc0 sp=0xc008cf6f28 pc=0xc19546 google.golang.org/grpc.recv(0x6?, {0x7f275f301038, 0xc0000af170}, 0xc2b369?, {0x0?, 0x0?}, {0x51d3160, 0xc00e1c5bf0}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc008cf7038 sp=0xc008cf6fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc004366000, {0x51d3160?, 0xc00e1c5bf0}, 0x5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc008cf7140 sp=0xc008cf7038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc008cf7170 sp=0xc008cf7140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc007e72d80, 0xc008cf7248, 0xc008cf7218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc008cf71c8 sp=0xc008cf7170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc007e72d80, {0x51d3160?, 0xc00e1c5bf0?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc008cf7278 sp=0xc008cf71c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0018ab860, {0x51d3160?, 0xc00e1c5bf0?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc008cf72c0 sp=0xc008cf7278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc0070b01e0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc008cf72f0 sp=0xc008cf72c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc008cf7310 sp=0xc008cf72f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0072cfd10, 0xc008cf7780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc008cf73b8 sp=0xc008cf7310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc004771b80?}, {{0xc0045eeb90, 0x1, 0x8}, {0xc0045eeb98, 0x1, 0x8}}, {0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc008cf79a0 sp=0xc008cf73b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc004771b80}, 0xc0072cfc70, 0x5683b00?, {{0xc0045eeb90, 0x1, 0x8}, {0xc0045eeb98, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc008cf7e80 sp=0xc008cf79a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc004771b80?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc008cf7f58 sp=0xc008cf7e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc008cf7f78 sp=0xc008cf7f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc008cf7fe0 sp=0xc008cf7f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008cf7fe8 sp=0xc008cf7fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1216 [select, 9 minutes]: runtime.gopark(0xc0032b77b0?, 0x2?, 0x0?, 0xc5?, 0xc0032b7794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b7618 sp=0xc0032b75f8 pc=0x4907d6 runtime.selectgo(0xc0032b77b0, 0xc0032b7790, 0xc0032b77d0?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b7758 sp=0xc0032b7618 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0032b77e0 sp=0xc0032b7758 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b77e8 sp=0xc0032b77e0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 1217 [select, 9 minutes]: runtime.gopark(0xc0032b7fb0?, 0x2?, 0x80?, 0x1b?, 0xc0032b7f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b7e28 sp=0xc0032b7e08 pc=0x4907d6 runtime.selectgo(0xc0032b7fb0, 0xc0032b7f98, 0xc002a40360?, 0x0, 0x10000c006e84df0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b7f68 sp=0xc0032b7e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0032b7fe0 sp=0xc0032b7f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b7fe8 sp=0xc0032b7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 120469 [runnable]: runtime.Gosched(...) GOROOT/src/runtime/proc.go:318 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00d119500) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:570 +0x1f6 fp=0xc0030fdf88 sp=0xc0030fdf30 pc=0xbd7156 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc0030fdfe0 sp=0xc0030fdf88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0030fdfe8 sp=0xc0030fdfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 1219 [select]: runtime.gopark(0xc005ae7ca0?, 0x5?, 0x1d?, 0x0?, 0xc005ae7a66?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005ae7898 sp=0xc005ae7878 pc=0x4907d6 runtime.selectgo(0xc005ae7ca0, 0xc005ae7a5c, 0xc0048e40c0?, 0x0, 0xc0048e40c0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005ae79d8 sp=0xc005ae7898 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc004766e10, {0x624a8a0, 0xc0048e40c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:323 +0x95f fp=0xc005ae7d60 sp=0xc005ae79d8 pc=0x326315f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x624a948?, 0xc0048dea50?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x27a fp=0xc005ae7f30 sp=0xc005ae7d60 pc=0x326255a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005ae7fe0 sp=0xc005ae7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005ae7fe8 sp=0xc005ae7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1220 [semacquire, 9 minutes]: runtime.gopark(0xc006a31a68?, 0x46159f?, 0xe0?, 0xc1?, 0xc004766f30?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0034d99f0 sp=0xc0034d99d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc004f71590, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc0034d9a58 sp=0xc0034d99f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc0034d9a88 sp=0xc0034d9a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc006a31ae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc0034d9ab0 sp=0xc0034d9a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc004f71580) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc0034d9ac8 sp=0xc0034d9ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004f71580?, {0x624a8a0?, 0xc004f714c0?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc0034d9af8 sp=0xc0034d9ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc0048dec00?}, {0xc006a31c68, 0x2, 0xc006a31dc0?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc0034d9b48 sp=0xc0034d9af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001735600, {0x624a948, 0xc0048dec00}, 0xc0048e4140) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc0034d9e68 sp=0xc0034d9b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc0048deb70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc0034d9f30 sp=0xc0034d9e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0034d9fe0 sp=0xc0034d9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0034d9fe8 sp=0xc0034d9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 344615 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1b?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ee12f58 sp=0xc00ee12f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7450?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00ee12f90 sp=0xc00ee12f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7368, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00ee12fb0 sp=0xc00ee12f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc005023580?, 0xc0044aa000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00ee12fd8 sp=0xc00ee12fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc005023580, {0xc0044aa000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00ee13058 sp=0xc00ee12fd8 pc=0x535efa net.(*netFD).Read(0xc005023580, {0xc0044aa000?, 0x7b5a68?, 0xc005023580?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00ee130a0 sp=0xc00ee13058 pc=0x7a20c9 net.(*conn).Read(0xc007caa768, {0xc0044aa000?, 0x164290f?, 0xc007caa768?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00ee130e8 sp=0xc00ee130a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0158612c0, {0xc0044aa000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00ee13130 sp=0xc00ee130e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03bdaa798, {0xc0044aa000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00ee131d0 sp=0xc00ee13130 pc=0x370f896 bufio.(*Reader).fill(0xc00a043890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00ee13208 sp=0xc00ee131d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00a043890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00ee13228 sp=0xc00ee13208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00a043e68?, {0x62295c8, 0xc00a043890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00ee13260 sp=0xc00ee13228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00a043800, 0xc0005c79c0, 0xc00ee13442, 0xc00ee13444, {0x623b718, 0xc005768c60}, 0xc0059fa000, 0xc00e433294, 0xc00ee13440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00ee13398 sp=0xc00ee13260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00a043800, {0x624a948, 0xc005188150}, 0xc0005c7810, 0xc0059fa000, 0xc00b525f80, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00ee13768 sp=0xc00ee13398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc005188150}, {0x628a538, 0xc0158612c0}, {{{0xc0067ea009, 0x4}}, 0x1, 0xc00b525fb0, 0xc0051880f0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00ee138a8 sp=0xc00ee13768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc005188150}, {0x628a538?, 0xc0158612c0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00ee13e20 sp=0xc00ee138a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00b525ec0?}, {0x628a538, 0xc0158612c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00ee13eb0 sp=0xc00ee13e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00b525ec0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00ee13f30 sp=0xc00ee13eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ee13fe0 sp=0xc00ee13f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ee13fe8 sp=0xc00ee13fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 474768 [sleep]: runtime.gopark(0x4b8c67822ab?, 0xa0582?, 0xd5?, 0x73?, 0x80b0fd0028?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004201df0 sp=0xc004201dd0 pc=0x4907d6 time.Sleep(0x14697458a) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004201e30 sp=0xc004201df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00ab18270, 0xa0582, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004201e88 sp=0xc004201e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00ab18270, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004201ec8 sp=0xc004201e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc003fa3400?, 0x1c, 0xc0000adfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004201fa8 sp=0xc004201ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004201fe0 sp=0xc004201fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004201fe8 sp=0xc004201fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 3983 [select, 9 minutes]: runtime.gopark(0xc002a37508?, 0x2?, 0x0?, 0x38?, 0xc002a374f4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002a37368 sp=0xc002a37348 pc=0x4907d6 runtime.selectgo(0xc002a37508, 0xc002a374f0, 0xc001663c20?, 0x0, 0x23?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002a374a8 sp=0xc002a37368 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc00388fad0, {0x624a948, 0xc001663c20}, 0xc002a40360, {0x628ab10?, 0xc003450ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1b9 fp=0xc002a37560 sp=0xc002a374a8 pc=0x1d1a619 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc00742cb40, {0x628ab10, 0xc003450ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x92 fp=0xc002a375b0 sp=0xc002a37560 pc=0x1d18f12 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x51002e0?, 0xc00742cb40}, {0x627d570?, 0xc0030d3d60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:344 +0x9f fp=0xc002a375e8 sp=0xc002a375b0 pc=0x16c585f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x51002e0, 0xc00742cb40}, {0x627dc78?, 0xc003d52820?}, 0xc0016645b8, 0x5683fc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc002a37920 sp=0xc002a375e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc003d52820?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc002a37960 sp=0xc002a37920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x51002e0, 0xc00742cb40}, {0x627dc78, 0xc003d52820}, 0x1054f90?, 0xc001660e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc002a37990 sp=0xc002a37960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc003d52820?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc002a379d0 sp=0xc002a37990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc002a37a00 sp=0xc002a379d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc001663b90}, {0xc001835b08?, 0x46159f?}, 0xc001835ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc002a37a78 sp=0xc002a37a00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x51002e0, 0xc00742cb40}, {0x627dc78?, 0xc003d52820?}, 0xc0016645b8, 0xc001660e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc002a37b00 sp=0xc002a37a78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc003d52820?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc002a37b40 sp=0xc002a37b00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x51002e0, 0xc00742cb40}, {0x627dc78, 0xc003d52820}, 0xc0016645b8, 0x5683fc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc002a37b90 sp=0xc002a37b40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc004caa680}, 0xc0050846c0, 0xc000acf290, 0x8fc0b80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc002a37e48 sp=0xc002a37b90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc004caa680}, 0xc0050846c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc002a37f68 sp=0xc002a37e48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc002a37fe0 sp=0xc002a37f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002a37fe8 sp=0xc002a37fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 4119 [select]: runtime.gopark(0xc002d8df70?, 0x4?, 0x6?, 0x0?, 0xc002d8ded0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d8dd18 sp=0xc002d8dcf8 pc=0x4907d6 runtime.selectgo(0xc002d8df70, 0xc002d8dec8, 0x6213da0?, 0x0, 0xc0031e3bc0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d8de58 sp=0xc002d8dd18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc007f59ba0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc002d8dfc8 sp=0xc002d8de58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc002d8dfe0 sp=0xc002d8dfc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d8dfe8 sp=0xc002d8dfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 118146 [select]: runtime.gopark(0xc00df33db8?, 0x3?, 0x52?, 0x0?, 0xc00df33d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d5db90 sp=0xc002d5db70 pc=0x4907d6 runtime.selectgo(0xc002d5ddb8, 0xc00df33d2c, 0xc011b1e600?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d5dcd0 sp=0xc002d5db90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc011b1e600}, 0xc0109c39f0, {0xc005e769f0, 0x12}, 0xc003d24de0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc002d5de68 sp=0xc002d5dcd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc011b1e600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc002d5df30 sp=0xc002d5de68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002d5dfe0 sp=0xc002d5df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d5dfe8 sp=0xc002d5dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498353 [select]: runtime.gopark(0xc01257d208?, 0x2?, 0x80?, 0xa7?, 0xc01257d204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01257d090 sp=0xc01257d070 pc=0x4907d6 runtime.selectgo(0xc01257d208, 0xc01257d200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01257d1d0 sp=0xc01257d090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0065f9d40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01257d238 sp=0xc01257d1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ef82160, {0x51b9a20?, 0xc00418e700}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01257d340 sp=0xc01257d238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01257d370 sp=0xc01257d340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0065f9b00, 0xc01257d448, 0xc01257d418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01257d3c8 sp=0xc01257d370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0065f9b00, {0x51b9a20?, 0xc00418e700?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01257d478 sp=0xc01257d3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc001709740?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc01068e960}, {0x51b9a20, 0xc00418e700}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01257d4e0 sp=0xc01257d478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc001709740}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc01068e960}, {0x51b9a20, 0xc00418e700}, 0x4db9936a3f487a0b?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01257d5d0 sp=0xc01257d4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc001709740?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc01068e960?}, {0x51b9a20?, 0xc00418e700?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01257d670 sp=0xc01257d5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e3a88, {0x624a948, 0xc001709740}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01257d6f0 sp=0xc01257d670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e3a88?}}, {0x624a948, 0xc001709740}, 0xc01068e960, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01257d7c8 sp=0xc01257d6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xb0fb830?, {0x624a948?, 0xc001709740?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc00b0fb928?}) :1 +0x65 fp=0xc01257d818 sp=0xc01257d7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00559e4e0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01257dab8 sp=0xc01257d818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00559e4e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12b2c09, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01257dbf0 sp=0xc01257dab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12b2c09, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01257e338 sp=0xc01257dbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12b2c09, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01257ec38 sp=0xc01257e338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12b2c09, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc01257f4a0 sp=0xc01257ec38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12b2c09, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01257f8c0 sp=0xc01257f4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc01257f9f8 sp=0xc01257f8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00d93f620}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc01257fc00 sp=0xc01257f9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00d93f620}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc01257ff30 sp=0xc01257fc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01257ffe0 sp=0xc01257ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01257ffe8 sp=0xc01257ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3724 [select]: runtime.gopark(0xc004e4fe18?, 0x2?, 0xe8?, 0xb5?, 0xc004e4fdfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004e4fc68 sp=0xc004e4fc48 pc=0x4907d6 runtime.selectgo(0xc004e4fe18, 0xc004e4fdf8, 0xc00327aac0?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004e4fda8 sp=0xc004e4fc68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001766840, {0x624a8a0, 0xc001f7b740}, 0xc004e4ffd0?, 0xc006e07aa0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc004e4ff20 sp=0xc004e4fda8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc001f7b740?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc004e4ff58 sp=0xc004e4ff20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc004e4ff78 sp=0xc004e4ff58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc004e4ffe0 sp=0xc004e4ff78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004e4ffe8 sp=0xc004e4ffe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 418838 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x50d?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007d66f58 sp=0xc007d66f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f300948?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc007d66f90 sp=0xc007d66f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300860, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc007d66fb0 sp=0xc007d66f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00b0c2580?, 0xc00ffe4000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc007d66fd8 sp=0xc007d66fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00b0c2580, {0xc00ffe4000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc007d67058 sp=0xc007d66fd8 pc=0x535efa net.(*netFD).Read(0xc00b0c2580, {0xc00ffe4000?, 0x7b5a68?, 0xc00b0c2580?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc007d670a0 sp=0xc007d67058 pc=0x7a20c9 net.(*conn).Read(0xc0051687a0, {0xc00ffe4000?, 0x164290f?, 0xc0051687a0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc007d670e8 sp=0xc007d670a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00c8b6840, {0xc00ffe4000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc007d67130 sp=0xc007d670e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc006098870, {0xc00ffe4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc007d671d0 sp=0xc007d67130 pc=0x370f896 bufio.(*Reader).fill(0xc003863090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc007d67208 sp=0xc007d671d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003863090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc007d67228 sp=0xc007d67208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003863668?, {0x62295c8, 0xc003863090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc007d67260 sp=0xc007d67228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003863000, 0xc00dd3a080, 0xc007d67442, 0xc007d67444, {0x623b718, 0xc005ec2480}, 0xc0059fa000, 0xc00754d2d4, 0xc007d67440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc007d67398 sp=0xc007d67260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003863000, {0x624a948, 0xc004ccb500}, 0xc00dd3a070, 0xc0059fa000, 0xc004ccb260, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc007d67768 sp=0xc007d67398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc004ccb500}, {0x628a538, 0xc00c8b6840}, {{{0xc00d6bf038, 0x4}}, 0x1, 0xc004ccb320, 0xc004ccb350, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc007d678a8 sp=0xc007d67768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc004ccb500}, {0x628a538?, 0xc00c8b6840}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc007d67e20 sp=0xc007d678a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc004ccb1a0?}, {0x628a538, 0xc00c8b6840}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc007d67eb0 sp=0xc007d67e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc004ccb1a0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc007d67f30 sp=0xc007d67eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc007d67fe0 sp=0xc007d67f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007d67fe8 sp=0xc007d67fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 418771 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc015981640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0159815d0 sp=0xc0159815b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00b510010, 0x16b) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc015981618 sp=0xc0159815d0 pc=0x4be30c sync.(*Cond).Wait(0xc003b95c00?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc015981650 sp=0xc015981618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc003b96268) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc015981708 sp=0xc015981650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003e75300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc015981cb8 sp=0xc015981708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003e75300, {0x624a8a0, 0xc00b510040}, 0x4119cd4800000000?, 0xc017511dc0?, 0xc00faa3ac0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc015981d70 sp=0xc015981cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00b510040?}, {0xc0059fa000?}, 0x3?, 0xc006b4e978?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc015981e20 sp=0xc015981d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc015981fe0 sp=0xc015981e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc015981fe8 sp=0xc015981fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 1258 [select, 7 minutes]: runtime.gopark(0xc00398fef8?, 0x2?, 0xa?, 0x0?, 0xc00398fea4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dff9d08 sp=0xc00dff9ce8 pc=0x4907d6 runtime.selectgo(0xc00dff9ef8, 0xc00398fea0, 0x629a5d0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dff9e48 sp=0xc00dff9d08 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging.(*CaptureIndexUsageStatsLoggingScheduler).start.func1({0x624a948, 0xc0034e8720}) github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging/captured_index_usage_stats.go:144 +0xff fp=0xc00dff9f30 sp=0xc00dff9e48 pc=0x327091f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00dff9fe0 sp=0xc00dff9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dff9fe8 sp=0xc00dff9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 68244 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1b0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d1aaf58 sp=0xc00d1aaf38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f2433d0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00d1aaf90 sp=0xc00d1aaf58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f2432e8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00d1aafb0 sp=0xc00d1aaf90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc015e97900?, 0xc0116ea000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00d1aafd8 sp=0xc00d1aafb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc015e97900, {0xc0116ea000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00d1ab058 sp=0xc00d1aafd8 pc=0x535efa net.(*netFD).Read(0xc015e97900, {0xc0116ea000?, 0x7b5a68?, 0xc015e97900?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00d1ab0a0 sp=0xc00d1ab058 pc=0x7a20c9 net.(*conn).Read(0xc01185eb00, {0xc0116ea000?, 0x164290f?, 0xc01185eb00?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00d1ab0e8 sp=0xc00d1ab0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc004fd6720, {0xc0116ea000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00d1ab130 sp=0xc00d1ab0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc015d2eaf8, {0xc0116ea000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00d1ab1d0 sp=0xc00d1ab130 pc=0x370f896 bufio.(*Reader).fill(0xc0100d5490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00d1ab208 sp=0xc00d1ab1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0100d5490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00d1ab228 sp=0xc00d1ab208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0100d5a68?, {0x62295c8, 0xc0100d5490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00d1ab260 sp=0xc00d1ab228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0100d5400, 0xc000763140, 0xc00d1ab442, 0xc00d1ab444, {0x623b718, 0xc0110561b0}, 0xc0059fa000, 0xc00f942a94, 0xc00d1ab440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00d1ab398 sp=0xc00d1ab260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0100d5400, {0x624a948, 0xc013d54240}, 0xc0007630b0, 0xc0059fa000, 0xc013d54180, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00d1ab768 sp=0xc00d1ab398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc013d54240}, {0x628a538, 0xc004fd6720}, {{{0xc00398204c, 0x4}}, 0x1, 0xc013d541b0, 0xc013d541e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00d1ab8a8 sp=0xc00d1ab768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc013d54240}, {0x628a538?, 0xc004fd6720}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00d1abe20 sp=0xc00d1ab8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc008263f80?}, {0x628a538, 0xc004fd6720}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00d1abeb0 sp=0xc00d1abe20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc008263f80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00d1abf30 sp=0xc00d1abeb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00d1abfe0 sp=0xc00d1abf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d1abfe8 sp=0xc00d1abfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498110 [select]: runtime.gopark(0xc00d87be60?, 0x3?, 0x0?, 0x0?, 0xc00d87be32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d87bcb8 sp=0xc00d87bc98 pc=0x4907d6 runtime.selectgo(0xc00d87be60, 0xc00d87be2c, 0xc011e07860?, 0x0, 0xc004173ba0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00d87bdf8 sp=0xc00d87bcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc007b5d8d0, {0x624a948, 0xc011e07860}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00d87beb0 sp=0xc00d87bdf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc011e07860?}) :1 +0x39 fp=0xc00d87bed8 sp=0xc00d87beb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc011e07860}, {0x53ddbcd?, 0xc00462ff60?}, 0xc00462ffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00d87bf50 sp=0xc00d87bed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00d87bfe0 sp=0xc00d87bf50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d87bfe8 sp=0xc00d87bfe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 498456 [select]: runtime.gopark(0xc00fa1f208?, 0x2?, 0x88?, 0xa5?, 0xc00fa1f204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fa1f090 sp=0xc00fa1f070 pc=0x4907d6 runtime.selectgo(0xc00fa1f208, 0xc00fa1f200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fa1f1d0 sp=0xc00fa1f090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc011e31320) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00fa1f238 sp=0xc00fa1f1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018ac580, {0x51b9a20?, 0xc00d502100}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00fa1f340 sp=0xc00fa1f238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00fa1f370 sp=0xc00fa1f340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011e30ea0, 0xc00fa1f448, 0xc00fa1f418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00fa1f3c8 sp=0xc00fa1f370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011e30ea0, {0x51b9a20?, 0xc00d502100?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00fa1f478 sp=0xc00fa1f3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc002196c00?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0081db2c0}, {0x51b9a20, 0xc00d502100}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00fa1f4e0 sp=0xc00fa1f478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc002196c00}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081db2c0}, {0x51b9a20, 0xc00d502100}, 0x686f9832e17f73c3?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00fa1f5d0 sp=0xc00fa1f4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc002196c00?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0081db2c0?}, {0x51b9a20?, 0xc00d502100?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00fa1f670 sp=0xc00fa1f5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0122772a0, {0x624a948, 0xc002196c00}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00fa1f6f0 sp=0xc00fa1f670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0122772a0?}}, {0x624a948, 0xc002196c00}, 0xc0081db2c0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00fa1f7c8 sp=0xc00fa1f6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xba83830?, {0x624a948?, 0xc002196c00?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00ba83928?}) :1 +0x65 fp=0xc00fa1f818 sp=0xc00fa1f7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc003d2c660, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00fa1fab8 sp=0xc00fa1f818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc003d2c660, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2895d71, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00fa1fbf0 sp=0xc00fa1fab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2895d71, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00fa20338 sp=0xc00fa1fbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2895d71, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00fa20c38 sp=0xc00fa20338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2895d71, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00fa214a0 sp=0xc00fa20c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2895d71, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00fa218c0 sp=0xc00fa214a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00fa219f8 sp=0xc00fa218c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc001f091a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00fa21c00 sp=0xc00fa219f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc001f091a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00fa21f30 sp=0xc00fa21c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00fa21fe0 sp=0xc00fa21f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fa21fe8 sp=0xc00fa21fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 442147 [sleep]: runtime.gopark(0x4b89b3931f3?, 0x4c10db?, 0x4f?, 0xaa?, 0x80775a6cb3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00168fdf0 sp=0xc00168fdd0 pc=0x4907d6 time.Sleep(0x13833a593) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00168fe30 sp=0xc00168fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc006dd6e10, 0x4c10db, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00168fe88 sp=0xc00168fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc006dd6e10, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00168fec8 sp=0xc00168fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00505ea00?, 0x19, 0xc0034dffb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00168ffa8 sp=0xc00168fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00168ffe0 sp=0xc00168ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00168ffe8 sp=0xc00168ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 4114 [IO wait]: runtime.gopark(0xc003dd1d80?, 0xb?, 0x0?, 0x0?, 0xa59?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003dd1a80 sp=0xc003dd1a60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x3e?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc003dd1ab8 sp=0xc003dd1a80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbcda8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc003dd1ad8 sp=0xc003dd1ab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc004a54100?, 0xc004a64000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc003dd1b00 sp=0xc003dd1ad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc004a54100, {0xc004a64000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc003dd1b80 sp=0xc003dd1b00 pc=0x535efa net.(*netFD).Read(0xc004a54100, {0xc004a64000?, 0x50c7cc0?, 0x1?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc003dd1bc8 sp=0xc003dd1b80 pc=0x7a20c9 net.(*conn).Read(0xc0051685f8, {0xc004a64000?, 0xc002ebb400?, 0xc002ebb500?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc003dd1c10 sp=0xc003dd1bc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc004d8dec0, {0xc004b6a040, 0x9, 0xc015c7ed7c?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc003dd1c48 sp=0xc003dd1c10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc004d8dec0}, {0xc004b6a040, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc003dd1c90 sp=0xc003dd1c48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004b6a040?, 0x9?, 0xc002ebb470?}, {0x62052c0?, 0xc004d8dec0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc003dd1ce0 sp=0xc003dd1c90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc004b6a000) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc003dd1d90 sp=0xc003dd1ce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0045c65a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc003dd1fc8 sp=0xc003dd1d90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc003dd1fe0 sp=0xc003dd1fc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003dd1fe8 sp=0xc003dd1fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 351005 [sleep]: runtime.gopark(0x4b8b4d4af69?, 0x21cd8b?, 0x64?, 0x6f?, 0x808e65fbc1?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a971df0 sp=0xc00a971dd0 pc=0x4907d6 time.Sleep(0x135a3b63b) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a971e30 sp=0xc00a971df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00da9a030, 0x21cd8b, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a971e88 sp=0xc00a971e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00da9a030, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a971ec8 sp=0xc00a971e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc002ad3500?, 0x31, 0xc0036bcf70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a971fa8 sp=0xc00a971ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a971fe0 sp=0xc00a971fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a971fe8 sp=0xc00a971fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 3939 [runnable]: runtime.gopark(0x0?, 0xb?, 0x0?, 0x0?, 0x97e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b2baa0 sp=0xc001b2ba80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0xbef6f1?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc001b2bad8 sp=0xc001b2baa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbce98, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc001b2baf8 sp=0xc001b2bad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc003165a00?, 0xc004824000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001b2bb20 sp=0xc001b2baf8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc003165a00, {0xc004824000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc001b2bba0 sp=0xc001b2bb20 pc=0x535efa net.(*netFD).Read(0xc003165a00, {0xc004824000?, 0xc001b2bc70?, 0x46159f?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc001b2bbe8 sp=0xc001b2bba0 pc=0x7a20c9 net.(*conn).Read(0xc002ceaa30, {0xc004824000?, 0x7f278cfaa108?, 0x60?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc001b2bc30 sp=0xc001b2bbe8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00399d200, {0xc004824000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc001b2bc78 sp=0xc001b2bc30 pc=0x1641f96 bufio.(*Reader).Read(0xc0040e23c0, {0xc0049af000, 0x9, 0xc011f07320?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc001b2bcb0 sp=0xc001b2bc78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc0040e23c0}, {0xc0049af000, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc001b2bcf8 sp=0xc001b2bcb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc0049af000?, 0x9?, 0x84?}, {0x62052c0?, 0xc0040e23c0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc001b2bd48 sp=0xc001b2bcf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0049aefc0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc001b2bdf8 sp=0xc001b2bd48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0038e3380, 0xc001b2bfd0?, 0x1054c46?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc001b2bf18 sp=0xc001b2bdf8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc0038e3380}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc001b2bf80 sp=0xc001b2bf18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc001b2bfe0 sp=0xc001b2bf80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b2bfe8 sp=0xc001b2bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 3733 [select]: runtime.gopark(0xc0034ddc60?, 0xa?, 0xa0?, 0xa8?, 0xc0034dd8d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0034dd6e8 sp=0xc0034dd6c8 pc=0x4907d6 runtime.selectgo(0xc0034ddc60, 0xc0034dd8c0, 0xc011af8820?, 0x0, 0x16?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0034dd828 sp=0xc0034dd6e8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0077de900, {0x624a948?, 0xc002ba1950?}, 0xc45d72?, 0xc001c59e00, 0xc002a40360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:268 +0x51f fp=0xc0034ddef0 sp=0xc0034dd828 pc=0x1c85b3f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x624a948?, 0xc002ba1950?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:221 +0x3c fp=0xc0034ddf30 sp=0xc0034ddef0 pc=0x1c855dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0034ddfe0 sp=0xc0034ddf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0034ddfe8 sp=0xc0034ddfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 6145 [IO wait]: runtime.gopark(0xc003993558?, 0xb?, 0x0?, 0x0?, 0xc3f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003993540 sp=0xc003993520 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x8b261e?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc003993578 sp=0xc003993540 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f2435b8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc003993598 sp=0xc003993578 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0137bba80?, 0xc013781000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0039935c0 sp=0xc003993598 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0137bba80, {0xc013781000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc003993640 sp=0xc0039935c0 pc=0x535efa net.(*netFD).Read(0xc0137bba80, {0xc013781000?, 0xc0039936e0?, 0x8b2350?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc003993688 sp=0xc003993640 pc=0x7a20c9 net.(*conn).Read(0xc00ff53f18, {0xc013781000?, 0xc003993710?, 0x60d7e3?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0039936d0 sp=0xc003993688 pc=0x7b53c5 net/http.(*connReader).Read(0xc0137cca50, {0xc013781000, 0x1000, 0x1000}) GOROOT/src/net/http/server.go:786 +0x171 fp=0xc003993720 sp=0xc0039936d0 pc=0x8ad831 bufio.(*Reader).fill(0xc0137bea80) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc003993758 sp=0xc003993720 pc=0x5ae51f bufio.(*Reader).ReadSlice(0xc0137bea80, 0x0?) GOROOT/src/bufio/bufio.go:372 +0x2f fp=0xc0039937a8 sp=0xc003993758 pc=0x5af10f bufio.(*Reader).ReadLine(0xc0137bea80) GOROOT/src/bufio/bufio.go:401 +0x27 fp=0xc0039937f8 sp=0xc0039937a8 pc=0x5af367 net/textproto.(*Reader).readLineSlice(0xc007276ae0) GOROOT/src/net/textproto/reader.go:58 +0x99 fp=0xc0039938e8 sp=0xc0039937f8 pc=0x844bb9 net/textproto.(*Reader).ReadLine(...) GOROOT/src/net/textproto/reader.go:39 net/http.readRequest(0xc00ff53f18?) GOROOT/src/net/http/request.go:1036 +0x79 fp=0xc0039939f0 sp=0xc0039938e8 pc=0x8a8bb9 net/http.(*conn).readRequest(0xc0137b32c0, {0x624a8a0, 0xc0137b8bc0}) GOROOT/src/net/http/server.go:994 +0x24a fp=0xc003993b78 sp=0xc0039939f0 pc=0x8aedca net/http.(*conn).serve(0xc0137b32c0, {0x624a948, 0xc0059919e0}) GOROOT/src/net/http/server.go:1916 +0x345 fp=0xc003993fb8 sp=0xc003993b78 pc=0x8b3305 net/http.(*Server).Serve.func3() GOROOT/src/net/http/server.go:3102 +0x2e fp=0xc003993fe0 sp=0xc003993fb8 pc=0x8b8cee runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003993fe8 sp=0xc003993fe0 pc=0x4c2401 created by net/http.(*Server).Serve GOROOT/src/net/http/server.go:3102 +0x4db goroutine 492498 [chan receive]: runtime.gopark(0x2faf080?, 0x3b9aca00?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc015a358d8 sp=0xc015a358b8 pc=0x4907d6 runtime.chanrecv(0xc004ea48a0, 0xc015a35b20, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc015a35968 sp=0xc015a358d8 pc=0x45a91b runtime.chanrecv1(0x1?, 0xc002125400?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc015a35990 sp=0xc015a35968 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1290 +0x127 fp=0xc015a35b58 sp=0xc015a35990 pc=0x19c9a07 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d78fca11, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1468 +0x17b9 fp=0xc015a363c0 sp=0xc015a35b58 pc=0x19c93d9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d78fca11, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc015a367e0 sp=0xc015a363c0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc010c82bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc015a36a38 sp=0xc015a367e0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc010c82b80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc015a36b60 sp=0xc015a36a38 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc010c82b50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc015a36df0 sp=0xc015a36b60 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc010c82a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc015a37390 sp=0xc015a36df0 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc010c82a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc015a37680 sp=0xc015a37390 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc010c82920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc015a37980 sp=0xc015a37680 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc010c82900?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc015a37b28 sp=0xc015a37980 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc010c82850, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc015a37d40 sp=0xc015a37b28 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc010c82680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc015a37f98 sp=0xc015a37d40 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc015a380c0 sp=0xc015a37f98 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc003430000, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc015a38388 sp=0xc015a380c0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc015a384a0 sp=0xc015a38388 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc004257b00, {0x624a948, 0xc00cc2c3c0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc015a386c0 sp=0xc015a384a0 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc004257b00, {0x624a948, 0xc00cc2c3c0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc015a389b8 sp=0xc015a386c0 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc004ea43c0, {0x624a948, 0xc00cc2c3c0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc015a38a98 sp=0xc015a389b8 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).nextKey(0xc0082fa000, {0x624a948?, 0xc00cc2c3c0?}) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:665 +0x7b fp=0xc015a38c48 sp=0xc015a38a98 pc=0x232b69b github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).startScan(...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:635 github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).StartScan(0xc0082fa000, {0x624a948, 0xc00cc2c3c0}, {0xc00f6c5500, 0x1?, 0x203004?}, {0xc005e69200?, 0x11?, 0x45e42d?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:492 +0x18f fp=0xc015a38cc0 sp=0xc015a38c48 pc=0x232a82f github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc0083e5500) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:999 +0xbca fp=0xc015a38ea0 sp=0xc015a38cc0 pc=0x258acca github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc0083e5500) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:731 +0x70 fp=0xc015a38f20 sp=0xc015a38ea0 pc=0x25899d0 github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc00f6c4000, {{0x0?, 0x0?}, 0xc00cc2c3f0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc015a38f98 sp=0xc015a38f20 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*errorIfRowsNode).Next(0xc01104fe20, {{0x624a948?, 0xc00cc2c360?}, 0xc00cb71c00?, 0xc003f06b78?}) github.com/cockroachdb/cockroach/pkg/sql/error_if_rows.go:42 +0x4b fp=0xc015a38fd0 sp=0xc015a38f98 pc=0x33e49cb github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc00f9f9b80) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc015a390c8 sp=0xc015a38fd0 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc006acc600) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:243 +0x13c fp=0xc015a39448 sp=0xc015a390c8 pc=0x299d6fc github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc006b120b0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc015a39460 sp=0xc015a39448 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc015a39478 sp=0xc015a39460 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc00409b880?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc015a394b8 sp=0xc015a39478 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc006b120b0, {0x624a8a0?, 0xc00c985f80?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc015a39648 sp=0xc015a394b8 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc006274d70, {0x624a8a0, 0xc00c985f80}, 0xc003430000?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc015a396e8 sp=0xc015a39648 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc0474476e0}, 0xc00723a0f0, 0xc003430000, 0xc003b8a980, 0xc00d4f7180, 0xc0067d8800, 0xc00db31dd0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc015a39f98 sp=0xc015a396e8 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).planAndRunPostquery(0xc0052d77c0, {0x624a948, 0xc0474476e0}, {{0x624c6d0?, 0xc01104fe20?}, 0x0?}, 0xc003f06b78, 0xc0067d8800, 0x33b057f?, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:2044 +0x71e fp=0xc015a3a148 sp=0xc015a39f98 pc=0x33bfdfe github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunCascadesAndChecks(0xc0052d77c0, {0x624a948, 0xc0474476e0}, 0xc003f06b78, 0xc013654740, 0xc003f072d8, 0xc00b9ce380) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1931 +0xe8c fp=0xc015a3a2f8 sp=0xc015a3a148 pc=0x33bf44c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc003f06600?, {0x624a948, 0xc0474476e0}, 0xc003f06eb0, 0xc003f06b78?, 0xc003f06b78, 0xc00b9ce380, 0xc013654740) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1488 +0x3aa fp=0xc015a3a410 sp=0xc015a3a2f8 pc=0x33bc68a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc003f06600, {0x624a948?, 0xc0474476e0}, 0xc003f06b78, 0xc0474476e0?, {0x7f275f329680?, 0xc002a176c8}, 0xa9?, 0xc00c3a50f8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc015a3a518 sp=0xc015a3a410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc003f06600, {0x624a8a0, 0xc001865b80}, 0xc003f06b78, {0x7f275f329680, 0xc002a176c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc015a3a938 sp=0xc015a3a518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc003f06600, {0x624a948?, 0xc01853e180?}, {{0x6273140, 0xc016632140}, {0xc005d58864, 0x448}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc015a3b2c8 sp=0xc015a3a938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc01853e180?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc015a3b380 sp=0xc015a3b2c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc01853e180?}, {0x6273140?, 0xc016632140?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc015a3b4d0 sp=0xc015a3b380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc003f06600, {0x624a948, 0xc01853e180}, {{0x6273140, 0xc016632140}, {0xc005d58864, 0x448}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc015a3b648 sp=0xc015a3b4d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6273140, 0xc016632140}, {0xc005d58864, 0x448}, 0x0, 0x1}, {0xc1116e81404f1846, 0x7f9aa37c94, 0x0}, {0xc1116e81404f1f75, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc015a3b708 sp=0xc015a3b648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003f06600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc015a3bcb8 sp=0xc015a3b708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003f06600, {0x624a8a0, 0xc00c2e2b00}, 0x4117f59400000000?, 0xc00cf53dc0?, 0xc011ce8220?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc015a3bd70 sp=0xc015a3bcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00c2e2b00?}, {0xc0059fa000?}, 0x3?, 0xc0053b01c8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc015a3be20 sp=0xc015a3bd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc015a3bfe0 sp=0xc015a3be20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc015a3bfe8 sp=0xc015a3bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498124 [select]: runtime.gopark(0xc00d6ffe60?, 0x3?, 0x0?, 0x0?, 0xc00d6ffe32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d6ffcb8 sp=0xc00d6ffc98 pc=0x4907d6 runtime.selectgo(0xc00d6ffe60, 0xc00d6ffe2c, 0xc010bc6750?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00d6ffdf8 sp=0xc00d6ffcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0146fd350, {0x624a948, 0xc010bc6750}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00d6ffeb0 sp=0xc00d6ffdf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc010bc6750?}) :1 +0x39 fp=0xc00d6ffed8 sp=0xc00d6ffeb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc010bc6750}, {0x53ddbcd?, 0x1054e20?}, 0xc00a57ffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00d6fff50 sp=0xc00d6ffed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00d6fffe0 sp=0xc00d6fff50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d6fffe8 sp=0xc00d6fffe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 121100 [select, 6 minutes]: runtime.gopark(0xc0174d5fb0?, 0x2?, 0x10?, 0x2e?, 0xc0174d5f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004a0fe18 sp=0xc004a0fdf8 pc=0x4907d6 runtime.selectgo(0xc004a0ffb0, 0xc0174d5f90, 0xc008cfe700?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004a0ff58 sp=0xc004a0fe18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc004a0ffe0 sp=0xc004a0ff58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004a0ffe8 sp=0xc004a0ffe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 3698 [semacquire, 9 minutes]: runtime.gopark(0x18?, 0xc000580400?, 0xa0?, 0xe2?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002a39aa8 sp=0xc002a39a88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc001e2f850, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc002a39b10 sp=0xc002a39aa8 pc=0x4a17de sync.runtime_Semacquire(0xc001e2f840?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc002a39b40 sp=0xc002a39b10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc002a39b68 sp=0xc002a39b40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc001e2f840) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc002a39b80 sp=0xc002a39b68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc001e2f840?, {0x624a8a0?, 0xc001e2f800?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc002a39bb0 sp=0xc002a39b80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc001e2f440}, {0xc001e2f7c0, 0x1, 0x2?}, 0x1, 0xc001e86f60, {0xc007473180, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc002a39d38 sp=0xc002a39bb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc002a10ed0?, {0x624a8a0, 0xc001e2f440}, {0xc0035f9c20, 0x1, 0xc001b2def0?}, {0x1e35dcf?, 0x1b2df00?, 0xc0?}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc002a39e58 sp=0xc002a39d38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1d74612?, {0x624a8a0?, 0xc001e2f440?}, {0xc0035f9c20?, 0xc000ba0f00?, 0xc001b2dfd0?}, {0x1054c46?, 0x624a948?, 0x0?}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc002a39ed0 sp=0xc002a39e58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc001e2f440?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc002a39f58 sp=0xc002a39ed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc002a39f78 sp=0xc002a39f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc002a39fe0 sp=0xc002a39f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002a39fe8 sp=0xc002a39fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 478056 [sleep]: runtime.gopark(0x4b9314c3081?, 0x400ea7?, 0xf5?, 0x39?, 0x811cdf5a4a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e1a3df0 sp=0xc00e1a3dd0 pc=0x4907d6 time.Sleep(0x147a5939c) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00e1a3e30 sp=0xc00e1a3df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0114d8180, 0x400ea7, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00e1a3e88 sp=0xc00e1a3e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0114d8180, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00e1a3ec8 sp=0xc00e1a3e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00c464280?, 0x9, 0xc003eaef70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00e1a3fa8 sp=0xc00e1a3ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00e1a3fe0 sp=0xc00e1a3fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e1a3fe8 sp=0xc00e1a3fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498449 [select]: runtime.gopark(0xc008863c18?, 0x3?, 0xd2?, 0x39?, 0xc008863b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0088638f0 sp=0xc0088638d0 pc=0x4907d6 runtime.selectgo(0xc008863c18, 0xc008863b34, 0xc00511cc80?, 0x0, 0xa?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008863a30 sp=0xc0088638f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc00564c0f0}, {0x6221560, 0xc00cd1d0e0}, {0x46?, {0xc008863e01?, 0x8863e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc008863da0 sp=0xc008863a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc00564c0f0?}, 0x16e, 0xc00cd1d0e0, 0x0?, 0xc000a71360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc008863e70 sp=0xc008863da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc00564c0f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc008863f30 sp=0xc008863e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc008863fe0 sp=0xc008863f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008863fe8 sp=0xc008863fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 396804 [select]: runtime.gopark(0xc00ca7b7f8?, 0x7?, 0x68?, 0x0?, 0xc00ca7b202?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ca7ae58 sp=0xc00ca7ae38 pc=0x4907d6 runtime.selectgo(0xc00ca7b7f8, 0xc00ca7b1f4, 0xc004959320?, 0x0, 0xc011cb1b00?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ca7af98 sp=0xc00ca7ae58 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc004959320, {0x624a948, 0xc011e015c0}, 0xc002cf01e0, 0xc00bb70700, 0xc01d62cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:569 +0x68f fp=0xc00ca7b878 sp=0xc00ca7af98 pc=0x19b910f github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x624a948?, 0xc011e015c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:515 +0x52 fp=0xc00ca7b8c8 sp=0xc00ca7b878 pc=0x19b8912 runtime/pprof.Do({0x624a948?, 0xc011e014a0?}, {{0xc011cb1a00?, 0x624a948?, 0xc011e014a0?}}, 0xc00ca7bb70) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00ca7b938 sp=0xc00ca7b8c8 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc004959320, {0x624a948, 0xc011e014a0}, 0xc002cf01e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:514 +0xb25 fp=0xc00ca7bc00 sp=0xc00ca7b938 pc=0x19b87c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc00b56cf00, {0x624a948, 0xc011e014a0}, {0x0, {0x17600a76e195bc1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:348 +0x9f fp=0xc00ca7bc40 sp=0xc00ca7bc00 pc=0x1cbd8bf github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc011e014a0}, 0xc013e8b040, 0xc010cded90?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc00ca7c0b0 sp=0xc00ca7bc40 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc011e014a0}, 0xc013e8b040?, {0x0, {0x17600a76e195bc1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc00ca7c1f0 sp=0xc00ca7c0b0 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc011e014a0}, 0xc0107d0a50, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc00ca7c510 sp=0xc00ca7c1f0 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00ca7c8d8 sp=0xc00ca7c510 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00ca7d260 sp=0xc00ca7c8d8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00ca7d4a8 sp=0xc00ca7d260 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc011e01440}, {0xc0086a7bb8?}, 0xc0107d0870) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00ca7db18 sp=0xc00ca7d4a8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc011e013b0}, 0xc0107d0870) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00ca7dbc0 sp=0xc00ca7db18 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc011e013b0?}, {0x52f0080?, 0xc0107d0870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00ca7dbf0 sp=0xc00ca7dbc0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc011e013b0}, {0x52f0080, 0xc0107d0870}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00ca7df20 sp=0xc00ca7dbf0 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011e013b0?}, {0x52f0080?, 0xc0107d0870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00ca7df60 sp=0xc00ca7df20 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc011e013b0}, {0x52f0080, 0xc0107d0870}, 0xc00ca7dff8?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00ca7df90 sp=0xc00ca7df60 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011e013b0?}, {0x52f0080?, 0xc0107d0870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00ca7dfd0 sp=0xc00ca7df90 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc011e013b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00ca7e008 sp=0xc00ca7dfd0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc011e013b0}, {0x1862d16?, 0xc00d99c500?}, 0xc00ca7e0c0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00ca7e080 sp=0xc00ca7e008 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc011e013b0?}, {0x52f0080?, 0xc0107d0870?}, 0x0?, 0xc008da54a0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00ca7e0f8 sp=0xc00ca7e080 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011e013b0?}, {0x52f0080?, 0xc0107d0870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00ca7e138 sp=0xc00ca7e0f8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc011e013b0?}, {0xc011e013b0?, 0x4?}, {0x52f0080?, 0xc0107d0870?}, {0x51b9a20?, 0xc0057f9200}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00ca7e178 sp=0xc00ca7e138 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011e013b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0107d0870}, {0x51b9a20, 0xc0057f9200}, 0xc00ca7e2f0?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00ca7e268 sp=0xc00ca7e178 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc011e013b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0107d0870}, {0x51b9a20, 0xc0057f9200}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00ca7e300 sp=0xc00ca7e268 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc011e01350}, 0xc0107d0870, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00ca7e380 sp=0xc00ca7e300 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x0?, {0x624a948?, 0xc011e01350?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00ca7e408 sp=0xc00ca7e380 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0119443c0, {_, _}, _, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00ca7e6a8 sp=0xc00ca7e408 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0119443c0, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00ca7e7e0 sp=0xc00ca7e6a8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00ca7ef28 sp=0xc00ca7e7e0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00ca7f828 sp=0xc00ca7ef28 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00ca80090 sp=0xc00ca7f828 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x17600a76e195d37c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00ca804b0 sp=0xc00ca80090 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00ca805e8 sp=0xc00ca804b0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00ca80710 sp=0xc00ca805e8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0xc0075b3c00?, {_, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc00ca80830 sp=0xc00ca80710 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e195bc1a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00ca80940 sp=0xc00ca80830 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc00b525aa0}, 0x0?, 0xc00c34b080) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00ca80b38 sp=0xc00ca80940 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc00b525aa0}, 0x19?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc00ca80b80 sp=0xc00ca80b38 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0005593b0, {0x624a948, 0xc00b525aa0}, 0xc00ca810f0, {{0x17600a76ad14c95b, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:395 +0x6fb fp=0xc00ca80ff0 sp=0xc00ca80b80 pc=0x1ca4d3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0xc00d9ce900?, {0x624a948, 0xc00b525aa0}, 0xc0152471b0, {{0x17600a76ad14c95b, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x145 fp=0xc00ca81200 sp=0xc00ca80ff0 pc=0x1ca4505 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc00b56cf50, {0x624a948, 0xc00b525aa0}, {0xc0083379e0, {0x17600a768f47645b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:519 +0x596 fp=0xc00ca817b8 sp=0xc00ca81200 pc=0x1cd0d36 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc00b525aa0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc00ca818e8 sp=0xc00ca817b8 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc00b56cf50, {0x624a948, 0xc00b525aa0}, {0xc0083379e0, {0x17600a768f47645b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc00ca81fd0 sp=0xc00ca818e8 pc=0x1ccefa6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc00b525aa0}, 0xc006649900, 0xc00ca82570?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc00ca82440 sp=0xc00ca81fd0 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00b525aa0}, 0xc00ca82628?, {0xc0083379e0, {0x17600a768f47645b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc00ca82580 sp=0xc00ca82440 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00b525aa0}, 0xc00bd4c870, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc00ca828a0 sp=0xc00ca82580 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a768f47645b, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00ca82c68 sp=0xc00ca828a0 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a768f47645b, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00ca835f0 sp=0xc00ca82c68 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x624a948?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00ca83838 sp=0xc00ca835f0 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00b525a40}, {0x203001?}, 0xc00bd4c780) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00ca83ea8 sp=0xc00ca83838 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00b5259b0}, 0xc00bd4c780) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00ca83f50 sp=0xc00ca83ea8 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00b5259b0?}, {0x52f0080?, 0xc00bd4c780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00ca83f80 sp=0xc00ca83f50 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00b5259b0}, {0x52f0080, 0xc00bd4c780}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00ca842b0 sp=0xc00ca83f80 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00b5259b0?}, {0x52f0080?, 0xc00bd4c780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00ca842f0 sp=0xc00ca842b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00b5259b0}, {0x52f0080, 0xc00bd4c780}, 0xc00ca84388?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00ca84320 sp=0xc00ca842f0 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00b5259b0?}, {0x52f0080?, 0xc00bd4c780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00ca84360 sp=0xc00ca84320 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00b5259b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00ca84398 sp=0xc00ca84360 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00b5259b0}, {0xc0037d2600?, 0x230f25c?}, 0xc00ca84450) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00ca84410 sp=0xc00ca84398 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00b5259b0?}, {0x52f0080?, 0xc00bd4c780?}, 0x0?, 0x300?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00ca84488 sp=0xc00ca84410 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00b5259b0?}, {0x52f0080?, 0xc00bd4c780?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00ca844c8 sp=0xc00ca84488 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00b5259b0?}, {0xc00b5259b0?, 0x4?}, {0x52f0080?, 0xc00bd4c780?}, {0x51b9a20?, 0xc000aee900}, 0x203002?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00ca84508 sp=0xc00ca844c8 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00b5259b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00bd4c780}, {0x51b9a20, 0xc000aee900}, 0xc00ca84680?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00ca845f8 sp=0xc00ca84508 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00b5259b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00bd4c780}, {0x51b9a20, 0xc000aee900}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00ca84690 sp=0xc00ca845f8 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00b525950}, 0xc00bd4c780, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00ca84710 sp=0xc00ca84690 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x532ec29?, {0x624a948?, 0xc00b525950?}, 0xc00ca84808?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00ca84798 sp=0xc00ca84710 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00ea7a120, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00ca84a38 sp=0xc00ca84798 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00ea7a120, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00ca84b70 sp=0xc00ca84a38 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00ca852b8 sp=0xc00ca84b70 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00ca85bb8 sp=0xc00ca852b8 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00ca86420 sp=0xc00ca85bb8 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76921bb735, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00ca86840 sp=0xc00ca86420 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00d1d0638, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00ca86a98 sp=0xc00ca86840 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00d1d0600, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00ca86bc0 sp=0xc00ca86a98 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00d1d05d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc00ca86e50 sp=0xc00ca86bc0 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00d1d04d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc00ca873f0 sp=0xc00ca86e50 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00d1d04d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc00ca876e0 sp=0xc00ca873f0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00d1d03a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc00ca879e0 sp=0xc00ca876e0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00d1d0380?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc00ca87b88 sp=0xc00ca879e0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00d1d02d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc00ca87da0 sp=0xc00ca87b88 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00d1d0100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc00ca87ff8 sp=0xc00ca87da0 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00ca88120 sp=0xc00ca87ff8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc015ad0c60, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc00ca883e8 sp=0xc00ca88120 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc00ca88500 sp=0xc00ca883e8 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00447fc80, {0x624a948, 0xc00b5256e0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc00ca88720 sp=0xc00ca88500 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00447fc80, {0x624a948, 0xc00b5256e0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc00ca88a18 sp=0xc00ca88720 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00c7f0540, {0x624a948, 0xc00b5256e0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc00ca88af8 sp=0xc00ca88a18 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0152b0a80, {0x624a948, 0xc00b5256e0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc00ca88dd0 sp=0xc00ca88af8 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0143466e0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc00ca88e28 sp=0xc00ca88dd0 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0057e1580) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc00ca88e40 sp=0xc00ca88e28 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc002d6e780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc00ca88e68 sp=0xc00ca88e40 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc002d6e780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc00ca88e80 sp=0xc00ca88e68 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc00ca88e98 sp=0xc00ca88e80 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x8?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc00ca88ed8 sp=0xc00ca88e98 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc002d6e780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc00ca88f10 sp=0xc00ca88ed8 pc=0x29a100c github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*noopProcessor).Next(0xc00ce6b680) github.com/cockroachdb/cockroach/pkg/sql/rowexec/noop.go:89 +0x46 fp=0xc00ca88f48 sp=0xc00ca88f10 pc=0x2595586 github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc0037d2600, {{0x624a948?, 0xc00b5255f0?}, 0x1000000000005?, 0xc00ca89048?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc00ca88fc0 sp=0xc00ca88f48 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00c9d6300, {{0x624a948, 0xc00b5255f0}, 0xc001d4cc00, 0xc003457178}) github.com/cockroachdb/cockroach/pkg/sql/update.go:164 +0xc4 fp=0xc00ca89020 sp=0xc00ca88fc0 pc=0x352d844 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 3730 [select]: runtime.gopark(0xc00383b168?, 0x3?, 0x20?, 0xf0?, 0xc00383b102?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00383af78 sp=0xc00383af58 pc=0x4907d6 runtime.selectgo(0xc00383b168, 0xc00383b0fc, 0xc00383b138?, 0x0, 0xc00383b130?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00383b0b8 sp=0xc00383af78 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc000199b20, {0x624a948, 0xc002ba0d20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1217 +0xe8 fp=0xc00383b1a8 sp=0xc00383b0b8 pc=0x165bee8 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc000199b20?, 0x5684220?}, {0x624a948?, 0xc002ba0d20?}, {0x51d3160, 0xc0134d2960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1253 +0x45 fp=0xc00383b1e0 sp=0xc00383b1a8 pc=0x165c0c5 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1301 github.com/cockroachdb/cockroach/pkg/rpc.(*clientStream).RecvMsg(0x0?, {0x51d3160?, 0xc0134d2960?}) :1 +0x77 fp=0xc00383b248 sp=0xc00383b1e0 pc=0x1666f77 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0044698c0, {0x51d3160?, 0xc0134d2960?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc00383b290 sp=0xc00383b248 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x627d528?, 0xc0044698c0?}}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1028 +0x59 fp=0xc00383b2d0 sp=0xc00383b290 pc=0x165b0b9 github.com/cockroachdb/cockroach/pkg/rpc.(*rangeFeedClientAdapter).Recv(0xc00383b300?) :1 +0x34 fp=0xc00383b2f0 sp=0xc00383b2d0 pc=0x1666a34 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc00383b310 sp=0xc00383b2f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0045bbc20, 0xc00383b780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc00383b3b8 sp=0xc00383b310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc001e2f800?}, {{0xc0035cc838, 0x2, 0x8}, {0xc0035cc840, 0x2, 0x8}}, {0x176009f82153225c, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc00383b9a0 sp=0xc00383b3b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc001e2f800}, 0xc0045bbb80, 0xc000ba1180?, {{0xc0035cc838, 0x2, 0x8}, {0xc0035cc840, 0x2, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc00383be80 sp=0xc00383b9a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc001e2f800?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc00383bf58 sp=0xc00383be80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00383bf78 sp=0xc00383bf58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00383bfe0 sp=0xc00383bf78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00383bfe8 sp=0xc00383bfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 4118 [select]: runtime.gopark(0xc008e5bf00?, 0x2?, 0x88?, 0xbd?, 0xc008e5bed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b787d60 sp=0xc00b787d40 pc=0x4907d6 runtime.selectgo(0xc00b787f00, 0xc008e5bed0, 0x0?, 0x0, 0xc008e5bef0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b787ea0 sp=0xc00b787d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00749e640, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc00b787f30 sp=0xc00b787ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0066e8ba0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc00b787f88 sp=0xc00b787f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc00b787fe0 sp=0xc00b787f88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b787fe8 sp=0xc00b787fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 482541 [sleep]: runtime.gopark(0x4b8ece4f779?, 0x400ebb?, 0xf2?, 0x36?, 0x80dca58d52?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a6fbdf0 sp=0xc00a6fbdd0 pc=0x4907d6 time.Sleep(0x14bd2ffad) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a6fbe30 sp=0xc00a6fbdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0196279b0, 0x400ebb, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a6fbe88 sp=0xc00a6fbe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0196279b0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a6fbec8 sp=0xc00a6fbe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00143ef80?, {0xc010a2d180?, 0x5, 0xc00143ef70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a6fbfa8 sp=0xc00a6fbec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a6fbfe0 sp=0xc00a6fbfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a6fbfe8 sp=0xc00a6fbfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 4022 [select, 9 minutes]: runtime.gopark(0xc002737578?, 0x2?, 0x0?, 0x4?, 0xc00273753c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0027373a8 sp=0xc002737388 pc=0x4907d6 runtime.selectgo(0xc002737578, 0xc002737538, 0xc0051a0f40?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0027374e8 sp=0xc0027373a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000c62700, {0x628a958, 0xc00605dd40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:313 +0x208 fp=0xc0027375b0 sp=0xc0027374e8 pc=0x1d82b08 github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x525f4e0?, 0xc000c62700}, {0x627d570?, 0xc003365dc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:244 +0x9f fp=0xc0027375e8 sp=0xc0027375b0 pc=0x1d4b57f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc0051bc000?}, 0xc0051814e8, 0x5683c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc002737920 sp=0xc0027375e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc0051bc000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc002737960 sp=0xc002737920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc0051bc000}, 0x1054f90?, 0xc0051a0f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc002737990 sp=0xc002737960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc0051bc000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc0027379d0 sp=0xc002737990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc002737a00 sp=0xc0027379d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0051b3950}, {0xc00143eb08?, 0x46159f?}, 0xc00143eac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc002737a78 sp=0xc002737a00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc0051bc000?}, 0xc0051814e8, 0xc0051a0f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc002737b00 sp=0xc002737a78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc0051bc000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc002737b40 sp=0xc002737b00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc0051bc000}, 0xc0051814e8, 0x5683c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc002737b90 sp=0xc002737b40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00512d7a0, 0xc000ace270, 0x8ff1fe0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc002737e48 sp=0xc002737b90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00512d7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc002737f68 sp=0xc002737e48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc002737fe0 sp=0xc002737f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002737fe8 sp=0xc002737fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498044 [select]: runtime.gopark(0xc00bdfbe60?, 0x3?, 0x0?, 0x0?, 0xc00bdfbe32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bdfbcb8 sp=0xc00bdfbc98 pc=0x4907d6 runtime.selectgo(0xc00bdfbe60, 0xc00bdfbe2c, 0xc00e89d200?, 0x0, 0xc0085cde40?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bdfbdf8 sp=0xc00bdfbcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc017452cd0, {0x624a948, 0xc00e89d200}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00bdfbeb0 sp=0xc00bdfbdf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc00e89d200?}) :1 +0x39 fp=0xc00bdfbed8 sp=0xc00bdfbeb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc00e89d200}, {0x53ddbcd?, 0x1054e20?}, 0xc0085cdfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00bdfbf50 sp=0xc00bdfbed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00bdfbfe0 sp=0xc00bdfbf50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bdfbfe8 sp=0xc00bdfbfe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 403955 [sleep]: runtime.gopark(0x4b8f4a10c2a?, 0x400e92?, 0x11?, 0xe3?, 0x80ea94396c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a701df0 sp=0xc00a701dd0 pc=0x4907d6 time.Sleep(0x152059699) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a701e30 sp=0xc00a701df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc022f19ce0, 0x400e92, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a701e88 sp=0xc00a701e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc022f19ce0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a701ec8 sp=0xc00a701e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00da27f80?, {0xc00ff28a80?, 0x3e, 0xc00da27f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a701fa8 sp=0xc00a701ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a701fe0 sp=0xc00a701fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a701fe8 sp=0xc00a701fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 1281 [select, 9 minutes]: runtime.gopark(0xc001402fb0?, 0x2?, 0x0?, 0x0?, 0xc001402f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001402e28 sp=0xc001402e08 pc=0x4907d6 runtime.selectgo(0xc001402fb0, 0xc001402f98, 0x6221ba0?, 0x0, 0x624a948?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001402f68 sp=0xc001402e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc001402fe0 sp=0xc001402f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001402fe8 sp=0xc001402fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 3984 [select]: runtime.gopark(0xc001835a10?, 0x2?, 0x53?, 0x64?, 0xc0018359f4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001835878 sp=0xc001835858 pc=0x4907d6 runtime.selectgo(0xc001835a10, 0xc0018359f0, 0xc003440800?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0018359b8 sp=0xc001835878 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc004d5c9b0, {0xc0016645b0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc001835a40 sp=0xc0018359b8 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc004d5c9b0, {0xc0016645b0?, 0xc000ab72d8?, 0xc001835a88?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc001835a80 sp=0xc001835a40 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001663aa0, {0xc0016645b0?, 0xc001835b00?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc001835ac8 sp=0xc001835a80 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc001663aa0}, {0xc0016645b0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc001835b10 sp=0xc001835ac8 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0050846c0, {0xc0016645b0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc001835b50 sp=0xc001835b10 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc0016645a0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc001835bd0 sp=0xc001835b50 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x1e63756?, 0xc0050846c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc001835c68 sp=0xc001835bd0 pc=0xc19546 google.golang.org/grpc.recv(0x1d1a426?, {0x7f275f301038, 0xc0000af170}, 0x2?, {0x0?, 0x0?}, {0x512ca20, 0xc00ad690e0}, 0x203002?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc001835ce0 sp=0xc001835c68 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc003d52820, {0x512ca20?, 0xc00ad690e0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc001835e10 sp=0xc001835ce0 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x9172428?, {0x512ca20?, 0xc00ad690e0?}) :1 +0x34 fp=0xc001835e38 sp=0xc001835e10 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc003450ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:363 +0x4c fp=0xc001835e68 sp=0xc001835e38 pc=0x16c594c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x624a948, 0xc001663c80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 fp=0xc001835f30 sp=0xc001835e68 pc=0x1d1a762 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001835fe0 sp=0xc001835f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001835fe8 sp=0xc001835fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1280 [select, 9 minutes]: runtime.gopark(0xc001853fb0?, 0x2?, 0x0?, 0x0?, 0xc001853f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001853e18 sp=0xc001853df8 pc=0x4907d6 runtime.selectgo(0xc001853fb0, 0xc001853f90, 0xc001853fd0?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001853f58 sp=0xc001853e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc001853fe0 sp=0xc001853f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001853fe8 sp=0xc001853fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 498170 [runnable]: runtime.gopark(0xae1b6659e46?, 0x0?, 0xa0?, 0xbb?, 0xc00a21a418?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a21a380 sp=0xc00a21a360 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x8?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00a21a3e8 sp=0xc00a21a380 pc=0x4a17de sync.runtime_SemacquireMutex(0xc0005ee310?, 0x48?, 0x4618e7?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00a21a418 sp=0xc00a21a3e8 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x0?, 0xc00a21a4e0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00a21a470 sp=0xc00a21a418 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x46b793?, 0x2000?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00a21a4b8 sp=0xc00a21a470 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004e6b340, {0xc00a21a5f8?, 0x4ccece?}, 0x0, 0x0, 0xc01407b710) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00a21a5b8 sp=0xc00a21a4b8 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00f209200, 0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc00a21a608 sp=0xc00a21a5b8 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc01407b710) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc00a21a690 sp=0xc00a21a608 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc002941970, 0x2f68ab0?, 0xc01407c640, {0x0?, 0xc01407b710?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc00a21a7c0 sp=0xc00a21a690 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x7f25833d1944?, 0x0?, 0xc00348dc50?, {0x0?, 0xc01407b710?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc00a21a810 sp=0xc00a21a7c0 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc01407c5f0?, 0xc005abd080?, {0x0?, 0xc01407b710?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc00a21a858 sp=0xc00a21a810 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc01407c5f0, 0xc005abd080, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc00a21a8b8 sp=0xc00a21a858 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc01407c5f0, {0xc00348dc50, 0x21, 0x21}, {0xc00348dbf0, 0x21, 0x21}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc00a21a908 sp=0xc00a21a8b8 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc01407b920, {0xc00348dbf0?, 0x0?, 0x1000000000000?}, 0x7f2763843758?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00a21aa38 sp=0xc00a21a908 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc01407b920, {0xc00348dc50?, 0x0?, 0x0?}, {0xc00348dbf0?, 0x4a59d2?, 0x21?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00a21aa78 sp=0xc00a21aa38 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc01407b788, {0xc00348dc50?, 0x0?, 0x21?}, {0xc00348dbf0, 0x21, 0x21}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00a21aad0 sp=0xc00a21aa78 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc01407b400, {0xc00348dbf0, 0x21, 0x21}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00a21ab68 sp=0xc00a21aad0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc013fb0588, {{0xc00348db90, 0x20, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00a21abb8 sp=0xc00a21ab68 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00af45200, {0x624a948, 0xc006e474d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc00a21ac20 sp=0xc00a21abb8 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc006e474d0}, {0x62b4fe0, 0xc013fb0588}, {0xc00348db90, 0x20, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc00a21b1f0 sp=0xc00a21ac20 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc006e474d0}, {0x628f760?, 0xc013fb0580?}, {0xc00348db90, 0x20, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc00a21b370 sp=0xc00a21b1f0 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc006e474d0?}, {0x628f760?, 0xc013fb0580?}, {0xc00348db90?, 0x34?, 0x14?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc00a21b460 sp=0xc00a21b370 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc016716bc0}, {{0x17600a76e1aff281, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_txn.go:71 +0x317 fp=0xc00a21b8a8 sp=0xc00a21b460 pc=0x1cfd357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e1aff281, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc00a21c138 sp=0xc00a21b8a8 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0107d12c0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00a21dfa8 sp=0xc00a21c138 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00a21ea18, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc00a21e618 sp=0xc00a21dfa8 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc00130a400, {0x624a948, 0xc006e474d0}, 0xc0107d12c0, 0xc015dfe640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc00a21f2a8 sp=0xc00a21e618 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00130a400, {0x624a948, 0xc006e474d0}, 0xc0107d12c0, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00a21f5c8 sp=0xc00a21f2a8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00130a400, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00a21f990 sp=0xc00a21f5c8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00a220318 sp=0xc00a21f990 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00a220560 sp=0xc00a220318 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc006e471d0}, {0xc0053a27b0?}, 0xc0107d11d0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00a220bd0 sp=0xc00a220560 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc006e463c0}, 0xc0107d11d0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00a220c78 sp=0xc00a220bd0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc006e463c0?}, {0x52f0080?, 0xc0107d11d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00a220ca8 sp=0xc00a220c78 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc006e463c0}, {0x52f0080, 0xc0107d11d0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00a220fd8 sp=0xc00a220ca8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc006e463c0?}, {0x52f0080?, 0xc0107d11d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a221018 sp=0xc00a220fd8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc006e463c0}, {0x52f0080, 0xc0107d11d0}, 0xc00a2210b0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00a221048 sp=0xc00a221018 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc006e463c0?}, {0x52f0080?, 0xc0107d11d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a221088 sp=0xc00a221048 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc006e463c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00a2210c0 sp=0xc00a221088 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc006e463c0}, {0xc00707f000?, 0xa?}, 0xc00a221178) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00a221138 sp=0xc00a2210c0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc006e463c0?}, {0x52f0080?, 0xc0107d11d0?}, 0xc00d1ad100?, 0x52a4f5?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00a2211b0 sp=0xc00a221138 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc006e463c0?}, {0x52f0080?, 0xc0107d11d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a2211f0 sp=0xc00a2211b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc006e463c0?}, {0xc006e463c0?, 0x4?}, {0x52f0080?, 0xc0107d11d0?}, {0x51b9a20?, 0xc0057f9780}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00a221230 sp=0xc00a2211f0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc006e463c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0107d11d0}, {0x51b9a20, 0xc0057f9780}, 0xc00a2213a8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00a221320 sp=0xc00a221230 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc006e463c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0107d11d0}, {0x51b9a20, 0xc0057f9780}, 0xc00a221488?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00a2213b8 sp=0xc00a221320 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc006e46240}, 0xc0107d11d0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00a221438 sp=0xc00a2213b8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc002698400?, {0x624a948?, 0xc006e46240?}, 0xc00d1ad4a8?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00a2214c0 sp=0xc00a221438 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0041e1b60, {_, _}, _, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00a221760 sp=0xc00a2214c0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0041e1b60, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00a221898 sp=0xc00a221760 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00a221fe0 sp=0xc00a221898 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00a2228e0 sp=0xc00a221fe0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00a223148 sp=0xc00a2228e0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x17600a76e1affce8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00a223568 sp=0xc00a223148 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00a2236a0 sp=0xc00a223568 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00a2237c8 sp=0xc00a2236a0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0x4b6f420?, {_, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc00a2238e8 sp=0xc00a2237c8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e1aff281, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00a2239f8 sp=0xc00a2238e8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc006fd8390}, 0x0?, 0xc00c34cc00) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00a223bf0 sp=0xc00a2239f8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc006fd8390}, 0x4596bd?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc00a223c38 sp=0xc00a223bf0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc00189b7a0, {0x624a948, 0xc006fd8390}, {{0xe8, 0x53, 0x5, 0x8c, 0x29, 0xd7, 0x44, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc00a223c88 sp=0xc00a223c38 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).startQueryPusherTxn.func1({0x624a948, 0xc006fd8390}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:854 +0x238 fp=0xc00a223f30 sp=0xc00a223c88 pc=0x19bbe78 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a223fe0 sp=0xc00a223f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a223fe8 sp=0xc00a223fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1279 [select]: runtime.gopark(0xc008d74d68?, 0x2?, 0x0?, 0x57?, 0xc008d74d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008d74bb8 sp=0xc008d74b98 pc=0x4907d6 runtime.selectgo(0xc008d74d68, 0xc008d74d30, 0x8?, 0x0, 0xc0231de000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008d74cf8 sp=0xc008d74bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc003b540a0, {0xc006076628, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc008d74d98 sp=0xc008d74cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc003b540a0, {0xc006076628?, 0xc002ca9668?, 0xc008d74de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc008d74dd8 sp=0xc008d74d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc008310f30, {0xc006076628?, 0xc008d74e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc008d74e20 sp=0xc008d74dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc008310f30}, {0xc006076628, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc008d74e68 sp=0xc008d74e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc003cf66c0, {0xc006076628, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc008d74ea8 sp=0xc008d74e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc006076618, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc008d74f28 sp=0xc008d74ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc004fbb028?, 0xc003cf66c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc008d74fc0 sp=0xc008d74f28 pc=0xc19546 google.golang.org/grpc.recv(0x6?, {0x7f275f301038, 0xc0000af170}, 0xc2b369?, {0x0?, 0x0?}, {0x51d3160, 0xc00e1c4ed0}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc008d75038 sp=0xc008d74fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc005096840, {0x51d3160?, 0xc00e1c4ed0}, 0x1?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc008d75140 sp=0xc008d75038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc008d75170 sp=0xc008d75140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc003cf6240, 0xc008d75248, 0xc008d75218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc008d751c8 sp=0xc008d75170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc003cf6240, {0x51d3160?, 0xc00e1c4ed0?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc008d75278 sp=0xc008d751c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc006e77f60, {0x51d3160?, 0xc00e1c4ed0?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc008d752c0 sp=0xc008d75278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc00be50d90) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc008d752f0 sp=0xc008d752c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc008d75310 sp=0xc008d752f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0009c9f90, 0xc008d75780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc008d753b8 sp=0xc008d75310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc004f71740?}, {{0xc0045ef3e0, 0x1, 0x8}, {0xc0045ef3e8, 0x1, 0x8}}, {0x176009f8267cb5ca, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc008d759a0 sp=0xc008d753b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc004f71740}, 0xc0009c9ef0, 0xc0045ef3e0?, {{0xc0045ef3e0, 0x1, 0x8}, {0xc0045ef3e8, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc008d75e80 sp=0xc008d759a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc004f71740?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc008d75f58 sp=0xc008d75e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc008d75f78 sp=0xc008d75f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc008d75fe0 sp=0xc008d75f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008d75fe8 sp=0xc008d75fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1277 [select, 9 minutes]: runtime.gopark(0xc001851e58?, 0x2?, 0x2?, 0x30?, 0xc001851dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001851c80 sp=0xc001851c60 pc=0x4907d6 runtime.selectgo(0xc001851e58, 0xc001851df8, 0xc004f71740?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001851dc0 sp=0xc001851c80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc004f71740}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc001851f58 sp=0xc001851dc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc001851f78 sp=0xc001851f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc001851fe0 sp=0xc001851f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001851fe8 sp=0xc001851fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 3712 [select, 9 minutes]: runtime.gopark(0xc001b2de58?, 0x2?, 0x0?, 0x30?, 0xc001b2ddfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b2dc80 sp=0xc001b2dc60 pc=0x4907d6 runtime.selectgo(0xc001b2de58, 0xc001b2ddf8, 0xc001e2f800?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001b2ddc0 sp=0xc001b2dc80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc001e2f800}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc001b2df58 sp=0xc001b2ddc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc001b2df78 sp=0xc001b2df58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc001b2dfe0 sp=0xc001b2df78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b2dfe8 sp=0xc001b2dfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 1276 [select]: runtime.gopark(0xc001852e18?, 0x2?, 0x88?, 0xb5?, 0xc001852dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001852c68 sp=0xc001852c48 pc=0x4907d6 runtime.selectgo(0xc001852e18, 0xc001852df8, 0xc0048e4140?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001852da8 sp=0xc001852c68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001735600, {0x624a8a0, 0xc004f714c0}, 0xc006e62ab0?, 0xc0032644e0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc001852f20 sp=0xc001852da8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc004f714c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc001852f58 sp=0xc001852f20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc001852f78 sp=0xc001852f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc001852fe0 sp=0xc001852f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001852fe8 sp=0xc001852fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498455 [select]: runtime.gopark(0xc00f8cf208?, 0x2?, 0x38?, 0xa4?, 0xc00f8cf204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f8cf090 sp=0xc00f8cf070 pc=0x4907d6 runtime.selectgo(0xc00f8cf208, 0xc00f8cf200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f8cf1d0 sp=0xc00f8cf090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc011e30d80) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00f8cf238 sp=0xc00f8cf1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018ac4d0, {0x51b9a20?, 0xc00d502080}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00f8cf340 sp=0xc00f8cf238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa4?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00f8cf370 sp=0xc00f8cf340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011e30a20, 0xc00f8cf448, 0xc00f8cf418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00f8cf3c8 sp=0xc00f8cf370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011e30a20, {0x51b9a20?, 0xc00d502080?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00f8cf478 sp=0xc00f8cf3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc002c49a40?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0081dae10}, {0x51b9a20, 0xc00d502080}, 0x0?, {0xc00a393c00, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00f8cf4e0 sp=0xc00f8cf478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc002c49a40}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081dae10}, {0x51b9a20, 0xc00d502080}, 0x526feaf16cdbb96b?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00f8cf5d0 sp=0xc00f8cf4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc002c49a40?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0081dae10?}, {0x51b9a20?, 0xc00d502080?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00f8cf670 sp=0xc00f8cf5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc012277280, {0x624a948, 0xc002c49a40}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00f8cf6f0 sp=0xc00f8cf670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc012277280?}}, {0x624a948, 0xc002c49a40}, 0xc0081dae10, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00f8cf7c8 sp=0xc00f8cf6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xba83830?, {0x624a948?, 0xc002c49a40?}, 0x624a948?, {0x0?, 0xc005751d20?, 0xc00ba83928?}) :1 +0x65 fp=0xc00f8cf818 sp=0xc00f8cf7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00559e120, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f8cfab8 sp=0xc00f8cf818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00559e120, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e286a15b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f8cfbf0 sp=0xc00f8cfab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e286a15b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f8d0338 sp=0xc00f8cfbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e286a15b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f8d0c38 sp=0xc00f8d0338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e286a15b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f8d14a0 sp=0xc00f8d0c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e286a15b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f8d18c0 sp=0xc00f8d14a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00f8d19f8 sp=0xc00f8d18c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc001f09170}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00f8d1c00 sp=0xc00f8d19f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc001f09170}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00f8d1f30 sp=0xc00f8d1c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f8d1fe0 sp=0xc00f8d1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f8d1fe8 sp=0xc00f8d1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1275 [semacquire, 9 minutes]: runtime.gopark(0x18?, 0x9bffb20?, 0xc0?, 0xb5?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00115baa8 sp=0xc00115ba88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc004f71810, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00115bb10 sp=0xc00115baa8 pc=0x4a17de sync.runtime_Semacquire(0xc004f71800?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00115bb40 sp=0xc00115bb10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00115bb68 sp=0xc00115bb40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc004f71800) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc00115bb80 sp=0xc00115bb68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004f71800?, {0x624a8a0?, 0xc004f71740?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc00115bbb0 sp=0xc00115bb80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc004f714c0}, {0xc004f71680, 0x1, 0x0?}, 0x0, 0xc0032644e0, {0xc0081ca250, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc00115bd38 sp=0xc00115bbb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc003c9c900?, {0x624a8a0, 0xc004f714c0}, {0xc0048deae0, 0x1, 0xc448e6?}, {0x100000001e35e6f?, 0x1?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc00115be58 sp=0xc00115bd38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc001851ee8?, {0x624a8a0?, 0xc004f714c0?}, {0xc0048deae0?, 0xc001851ee0?, 0xc001851fd0?}, {0x1054c46?, 0x624a948?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc00115bed0 sp=0xc00115be58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc004f714c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc00115bf58 sp=0xc00115bed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00115bf78 sp=0xc00115bf58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00115bfe0 sp=0xc00115bf78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00115bfe8 sp=0xc00115bfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498169 [chan receive]: runtime.gopark(0xc00cfbca10?, 0x49b1380?, 0xe0?, 0xdc?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0087aa790 sp=0xc0087aa770 pc=0x4907d6 runtime.chanrecv(0xc00d1bc480, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0087aa820 sp=0xc0087aa790 pc=0x45a91b runtime.chanrecv1(0x19bae6b?, 0x0?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc0087aa848 sp=0xc0087aa820 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:543 +0x3a fp=0xc0087aa870 sp=0xc0087aa848 pc=0x19baeda runtime.deferreturn() GOROOT/src/runtime/panic.go:476 +0x33 fp=0xc0087aa8b0 sp=0xc0087aa870 pc=0x48c7b3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc00189b7a0, {0x624a948, 0xc006fd82d0}, 0xc00af1a960, 0xc011dded40, 0xc010f0cfa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:606 +0x125b fp=0xc0087ab190 sp=0xc0087aa8b0 pc=0x19b9cdb github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x624a948?, 0xc006fd82d0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:515 +0x52 fp=0xc0087ab1e0 sp=0xc0087ab190 pc=0x19b8912 runtime/pprof.Do({0x624a948?, 0xc00cc2cff0?}, {{0xc00cfbc940?, 0x624a948?, 0xc00cc2cff0?}}, 0xc0087ab488) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0087ab250 sp=0xc0087ab1e0 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc00189b7a0, {0x624a948, 0xc00cc2cff0}, 0xc00af1a960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:514 +0xb25 fp=0xc0087ab518 sp=0xc0087ab250 pc=0x19b87c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc003b54ff0, {0x624a948, 0xc00cc2cff0}, {0x0, {0x17600a76d70e2387, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:348 +0x9f fp=0xc0087ab558 sp=0xc0087ab518 pc=0x1cbd8bf github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc003b54ff0, {0x624a948, 0xc00cc2cff0}, 0xc00750b680, 0xc000a73a10?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc0087ab9c8 sp=0xc0087ab558 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00cc2cff0}, 0xc00750b680?, {0x0, {0x17600a76d70e2387, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0087abb08 sp=0xc0087ab9c8 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00130a400, {0x624a948, 0xc00cc2cff0}, 0xc00723ac30, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0087abe28 sp=0xc0087abb08 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00130a400, {_, _}, {{{0x17600a76d70e2387, 0x0, 0x0}, 0x0, {0x17600a76d70e386f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc0087ac1f0 sp=0xc0087abe28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76d70e2387, 0x0, 0x0}, 0x0, {0x17600a76d70e386f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0087acb78 sp=0xc0087ac1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76d70e2387, 0x0, 0x0}, 0x0, {0x17600a76d70e386f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0087acdc0 sp=0xc0087acb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00cc2cf90}, {0x0?}, 0xc00723ab40) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0087ad430 sp=0xc0087acdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00cc2ce70}, 0xc00723ab40) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0087ad4d8 sp=0xc0087ad430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc00cc2ce70}, {0x52f0080?, 0xc00723ab40}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc0087ad518 sp=0xc0087ad4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00cc2ce70}, {0x52f0080, 0xc00723ab40}, 0xc011dde820, 0xc007892600) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0087ad848 sp=0xc0087ad518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00cc2ce70?}, {0x52f0080?, 0xc00723ab40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc0087ad888 sp=0xc0087ad848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00cc2ce70}, {0x52f0080, 0xc00723ab40}, 0xc0087ad920?, 0xc00cfbc640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0087ad8b8 sp=0xc0087ad888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00cc2ce70?}, {0x52f0080?, 0xc00723ab40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0087ad8f8 sp=0xc0087ad8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00cc2ce70?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0087ad930 sp=0xc0087ad8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00cc2ce70}, {0x203004?, 0x40?}, 0xc0087ad9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0087ad9a8 sp=0xc0087ad930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00cc2ce70?}, {0x52f0080?, 0xc00723ab40?}, 0x5003b00?, 0x7f278cfaa501?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0087ada20 sp=0xc0087ad9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00cc2ce70?}, {0x52f0080?, 0xc00723ab40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0087ada60 sp=0xc0087ada20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc00cc2ce70}, {0x52f0080, 0xc00723ab40}, 0xc011dde820, 0xc007892600) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc0087adab0 sp=0xc0087ada60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc00cc2ce70}, 0xc004ea56e0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc0087adb08 sp=0xc0087adab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00cd34b40, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc0087ade48 sp=0xc0087adb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00cd34b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc0087adf68 sp=0xc0087ade48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc0087adfe0 sp=0xc0087adf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0087adfe8 sp=0xc0087adfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 120662 [select, 6 minutes]: runtime.gopark(0xc003a73578?, 0x2?, 0x0?, 0x54?, 0xc003a7353c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003a733a8 sp=0xc003a73388 pc=0x4907d6 runtime.selectgo(0xc003a73578, 0xc003a73538, 0xc01103dc40?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc003a734e8 sp=0xc003a733a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000c62700, {0x628a958, 0xc012390f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:313 +0x208 fp=0xc003a735b0 sp=0xc003a734e8 pc=0x1d82b08 github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x525f4e0?, 0xc000c62700}, {0x627d570?, 0xc0124ce6c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:244 +0x9f fp=0xc003a735e8 sp=0xc003a735b0 pc=0x1d4b57f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc0172daf70?}, 0xc0080f0408, 0x5683c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc003a73920 sp=0xc003a735e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc0172daf70?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc003a73960 sp=0xc003a73920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc0172daf70}, 0x1054f90?, 0xc01103dc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc003a73990 sp=0xc003a73960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc0172daf70?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc003a739d0 sp=0xc003a73990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc003a73a00 sp=0xc003a739d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00cf8ecc0}, {0xc003a73b08?, 0x46159f?}, 0xc003a73ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc003a73a78 sp=0xc003a73a00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc0172daf70?}, 0xc0080f0408, 0xc01103dc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc003a73b00 sp=0xc003a73a78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc0172daf70?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc003a73b40 sp=0xc003a73b00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc0172daf70}, 0xc0080f0408, 0x5683c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc003a73b90 sp=0xc003a73b40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc01479a4e0}, 0xc01250e6c0, 0xc000ace270, 0x8ff1fe0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc003a73e48 sp=0xc003a73b90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc01479a4e0}, 0xc01250e6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc003a73f68 sp=0xc003a73e48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc003a73fe0 sp=0xc003a73f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003a73fe8 sp=0xc003a73fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 318371 [sleep]: runtime.gopark(0x4b899935c19?, 0x400eaf?, 0x4e?, 0xee?, 0x807523b09f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003949df0 sp=0xc003949dd0 pc=0x4907d6 time.Sleep(0x137a2becd) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003949e30 sp=0xc003949df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00f6c7080, 0x400eaf, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003949e88 sp=0xc003949e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00f6c7080, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003949ec8 sp=0xc003949e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc002d93f80?, {0xc00ac8f500?, 0x63, 0xc002d93f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003949fa8 sp=0xc003949ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003949fe0 sp=0xc003949fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003949fe8 sp=0xc003949fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 497382 [select]: runtime.gopark(0xc020520588?, 0x3?, 0x42?, 0x3?, 0xc0205204aa?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc020520260 sp=0xc020520240 pc=0x4907d6 runtime.selectgo(0xc020520588, 0xc0205204a4, 0xc00511d200?, 0x0, 0x5?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0205203a0 sp=0xc020520260 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc02a5ee8d0}, {0x6221538, 0xc01052bfc8}, {0x10?, {0xc003d965b0?, 0x58de870?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc020520710 sp=0xc0205203a0 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:888 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0xc00581a218?, {0x624a948, 0xc02a5ee8d0}, {{{0xc01121c818, 0x5, 0x8}, {0x0, 0x0, 0x0}}, {{0x4f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:881 +0x115 fp=0xc020520788 sp=0xc020520710 pc=0x1ca8435 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc00b56cf50, {0x624a948, 0xc02a5ee8d0}, {0xc00b07f320, {0x17600a7689741cde, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:655 +0x814 fp=0xc020520d40 sp=0xc020520788 pc=0x1cd0fb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc02a5ee8d0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc020520e70 sp=0xc020520d40 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc00b56cf50, {0x624a948, 0xc02a5ee8d0}, {0xc00b07f320, {0x17600a7689741cde, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc020521558 sp=0xc020520e70 pc=0x1ccefa6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc02a5ee8d0}, 0xc00c4f32c0, 0xc017ee3568?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc0205219c8 sp=0xc020521558 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc02a5ee8d0}, 0xc00c4f32c0?, {0xc00b07f320, {0x17600a7689741cde, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc020521b08 sp=0xc0205219c8 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc02a5ee8d0}, 0xc004f5e0f0, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc020521e28 sp=0xc020521b08 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a7689741cde, 0x0, 0x0}, 0x0, {0x17600a768a497ed5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc0205221f0 sp=0xc020521e28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a7689741cde, 0x0, 0x0}, 0x0, {0x17600a768a497ed5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc020522b78 sp=0xc0205221f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a768a497ed5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc020522dc0 sp=0xc020522b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc02a5ee810}, {0x0?}, 0xc004f5e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc020523430 sp=0xc020522dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc02a5ee690}, 0xc004f5e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0205234d8 sp=0xc020523430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc02a5ee690}, {0x52f0080?, 0xc004f5e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc020523518 sp=0xc0205234d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc02a5ee690}, {0x52f0080, 0xc004f5e000}, 0xc015900700, 0xc034a3dd88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc020523848 sp=0xc020523518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc02a5ee690?}, {0x52f0080?, 0xc004f5e000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc020523888 sp=0xc020523848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc02a5ee690}, {0x52f0080, 0xc004f5e000}, 0xc017ee5920?, 0xc00c62d800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0205238b8 sp=0xc020523888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc02a5ee690?}, {0x52f0080?, 0xc004f5e000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0205238f8 sp=0xc0205238b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc02a5ee690?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc020523930 sp=0xc0205238f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc02a5ee690}, {0x203005?, 0x40?}, 0xc017ee59e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0205239a8 sp=0xc020523930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc02a5ee690?}, {0x52f0080?, 0xc004f5e000?}, 0x5003b00?, 0x7f278cfaaa01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc020523a20 sp=0xc0205239a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc02a5ee690?}, {0x52f0080?, 0xc004f5e000?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc020523a60 sp=0xc020523a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc02a5ee690}, {0x52f0080, 0xc004f5e000}, 0xc015900700, 0xc034a3dd88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc020523ab0 sp=0xc020523a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc02a5ee690}, 0xc011d26f00, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc020523b08 sp=0xc020523ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc00b07e240, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc020523e48 sp=0xc020523b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc00b07e240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc020523f68 sp=0xc020523e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc020523fe0 sp=0xc020523f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc020523fe8 sp=0xc020523fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 3504 [select, 9 minutes]: runtime.gopark(0xc005b06fb0?, 0x2?, 0x0?, 0x0?, 0xc005b06f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b06e18 sp=0xc005b06df8 pc=0x4907d6 runtime.selectgo(0xc005b06fb0, 0xc005b06f90, 0xc005b06fd0?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005b06f58 sp=0xc005b06e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc005b06fe0 sp=0xc005b06f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b06fe8 sp=0xc005b06fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 3804 [select, 9 minutes]: runtime.gopark(0xc001839fb0?, 0x2?, 0x65?, 0x4b?, 0xc001839f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001839e28 sp=0xc001839e08 pc=0x4907d6 runtime.selectgo(0xc001839fb0, 0xc001839f98, 0xc002a40360?, 0x0, 0xc007473330?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001839f68 sp=0xc001839e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc001839fe0 sp=0xc001839f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001839fe8 sp=0xc001839fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 4011 [select, 9 minutes]: runtime.gopark(0xc005b27f98?, 0x2?, 0x0?, 0x0?, 0xc005b27f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001c6ee00 sp=0xc001c6ede0 pc=0x4907d6 runtime.selectgo(0xc001c6ef98, 0xc005b27f78, 0xc0070b00a0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001c6ef40 sp=0xc001c6ee00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0045feec0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc001c6efc8 sp=0xc001c6ef40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc001c6efe0 sp=0xc001c6efc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001c6efe8 sp=0xc001c6efe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 115792 [select]: runtime.gopark(0xc002c92e18?, 0x2?, 0x50?, 0x48?, 0xc002c92dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002c92c68 sp=0xc002c92c48 pc=0x4907d6 runtime.selectgo(0xc002c92e18, 0xc002c92df8, 0xc003f562c0?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002c92da8 sp=0xc002c92c68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc006663b80, {0x624a8a0, 0xc006c4e9c0}, 0xc010d81f50?, 0xc00376de60) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc002c92f20 sp=0xc002c92da8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc006c4e9c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc002c92f58 sp=0xc002c92f20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc002c92f78 sp=0xc002c92f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc002c92fe0 sp=0xc002c92f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002c92fe8 sp=0xc002c92fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 3799 [sync.Cond.Wait]: runtime.gopark(0xc00d165720?, 0xc01b81dc98?, 0x20?, 0x55?, 0xc01745b650?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ce35c78 sp=0xc00ce35c58 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc000b70b78, 0x1e00) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00ce35cc0 sp=0xc00ce35c78 pc=0x4be30c sync.(*Cond).Wait(0x5?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00ce35cf8 sp=0xc00ce35cc0 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000b70b60, {0x624a8a0, 0xc003745740}, 0xa18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb fp=0xc00ce35db0 sp=0xc00ce35cf8 pc=0x1d1cc5b github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x624a948?, 0xc0037fa690?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 fp=0xc00ce35f30 sp=0xc00ce35db0 pc=0x1d1dbd6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ce35fe0 sp=0xc00ce35f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ce35fe8 sp=0xc00ce35fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3699 [select]: runtime.gopark(0xc0049eee18?, 0x2?, 0x38?, 0xc8?, 0xc0049eedfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0049eec68 sp=0xc0049eec48 pc=0x4907d6 runtime.selectgo(0xc0049eee18, 0xc0049eedf8, 0xc003e9a8c0?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0049eeda8 sp=0xc0049eec68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0016c22c0, {0x624a8a0, 0xc001e2f440}, 0xc001ed1320?, 0xc001e86f60) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc0049eef20 sp=0xc0049eeda8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc001e2f440?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc0049eef58 sp=0xc0049eef20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc0049eef78 sp=0xc0049eef58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc0049eefe0 sp=0xc0049eef78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0049eefe8 sp=0xc0049eefe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 344616 [select]: runtime.gopark(0xc010d0f968?, 0x7?, 0x74?, 0x0?, 0xc010d0f372?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010d0efc8 sp=0xc010d0efa8 pc=0x4907d6 runtime.selectgo(0xc010d0f968, 0xc010d0f364, 0xc006eacb40?, 0x0, 0xc005af3b80?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc010d0f108 sp=0xc010d0efc8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc006eacb40, {0x624a948, 0xc00c841ef0}, 0xc0019c6d20, 0xc003dd8420, 0xc006eb02d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:569 +0x68f fp=0xc010d0f9e8 sp=0xc010d0f108 pc=0x19b910f github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x624a948?, 0xc00c841ef0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:515 +0x52 fp=0xc010d0fa38 sp=0xc010d0f9e8 pc=0x19b8912 runtime/pprof.Do({0x624a948?, 0xc00c841260?}, {{0xc005af3b40?, 0x624a948?, 0xc00c841260?}}, 0xc010d0fce0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc010d0faa8 sp=0xc010d0fa38 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc006eacb40, {0x624a948, 0xc00c841260}, 0xc0019c6d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:514 +0xb25 fp=0xc010d0fd70 sp=0xc010d0faa8 pc=0x19b87c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc006ef86e0, {0x624a948, 0xc00c841260}, {0x0, {0x17600a768c9fce21, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:348 +0x9f fp=0xc010d0fdb0 sp=0xc010d0fd70 pc=0x1cbd8bf github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc006ef86e0, {0x624a948, 0xc00c841260}, 0xc003279f40, 0xc006f08110?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc010d10220 sp=0xc010d0fdb0 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00c841260}, 0xc003279f40?, {0x0, {0x17600a768c9fce21, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc010d10360 sp=0xc010d10220 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc00c841260}, 0xc005078780, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc010d10680 sp=0xc010d10360 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc010d10a48 sp=0xc010d10680 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc010d113d0 sp=0xc010d10a48 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc010d11618 sp=0xc010d113d0 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00c841200}, {0x0?}, 0xc005078690) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc010d11c88 sp=0xc010d11618 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00c841170}, 0xc005078690) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc010d11d30 sp=0xc010d11c88 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00c841170?}, {0x52f0080?, 0xc005078690?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc010d11d60 sp=0xc010d11d30 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00c841170}, {0x52f0080, 0xc005078690}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc010d12090 sp=0xc010d11d60 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00c841170?}, {0x52f0080?, 0xc005078690?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010d120d0 sp=0xc010d12090 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00c841170}, {0x52f0080, 0xc005078690}, 0xc010d12168?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc010d12100 sp=0xc010d120d0 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00c841170?}, {0x52f0080?, 0xc005078690?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010d12140 sp=0xc010d12100 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00c841170?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc010d12178 sp=0xc010d12140 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00c841170}, {0x1000000000028?, 0x7f275a1576e0?}, 0xc010d12230) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc010d121f0 sp=0xc010d12178 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00c841170?}, {0x52f0080?, 0xc005078690?}, 0x40?, 0xc0052b3c00?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc010d12268 sp=0xc010d121f0 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00c841170?}, {0x52f0080?, 0xc005078690?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010d122a8 sp=0xc010d12268 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00c841170?}, {0xc00c841170?, 0x4?}, {0x52f0080?, 0xc005078690?}, {0x51b9a20?, 0xc0161dc180}, 0x203003?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc010d122e8 sp=0xc010d122a8 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00c841170}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc005078690}, {0x51b9a20, 0xc0161dc180}, 0xc010d12460?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc010d123d8 sp=0xc010d122e8 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00c841170}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc005078690}, {0x51b9a20, 0xc0161dc180}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc010d12470 sp=0xc010d123d8 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00c841110}, 0xc005078690, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc010d124f0 sp=0xc010d12470 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x1?, {0x624a948?, 0xc00c841110?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc010d12578 sp=0xc010d124f0 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0043afaa0, {_, _}, _, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010d12818 sp=0xc010d12578 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0043afaa0, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010d12950 sp=0xc010d12818 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010d13098 sp=0xc010d12950 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010d13998 sp=0xc010d13098 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc010d14200 sp=0xc010d13998 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x17600a768c9ff99e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc010d14620 sp=0xc010d14200 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc010d14758 sp=0xc010d14620 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc010d14880 sp=0xc010d14758 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0xc008c29180?, {_, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc010d149a0 sp=0xc010d14880 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a768c9fce21, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc010d14ab0 sp=0xc010d149a0 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc004c62ea0}, 0x0?, 0xc017454680) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc010d14ca8 sp=0xc010d14ab0 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc004c62ea0}, 0x19?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc010d14cf0 sp=0xc010d14ca8 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0005593b0, {0x624a948, 0xc004c62ea0}, 0xc010d15260, {{0x17600a768c9f9c51, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:395 +0x6fb fp=0xc010d15160 sp=0xc010d14cf0 pc=0x1ca4d3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0xc047494360?, {0x624a948, 0xc004c62ea0}, 0xc00d2ede18, {{0x17600a768c9f9c51, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x145 fp=0xc010d15370 sp=0xc010d15160 pc=0x1ca4505 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc006ef8730, {0x624a948, 0xc004c62ea0}, {0xc03be7aea0, {0x17600a7688712da3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:519 +0x596 fp=0xc010d15928 sp=0xc010d15370 pc=0x1cd0d36 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc004c62ea0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc010d15a58 sp=0xc010d15928 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc006ef8730, {0x624a948, 0xc004c62ea0}, {0xc03be7aea0, {0x17600a7688712da3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc010d16140 sp=0xc010d15a58 pc=0x1ccefa6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc006ef86e0, {0x624a948, 0xc004c62ea0}, 0xc00c4f2d20, 0xc010d16150?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc010d165b0 sp=0xc010d16140 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc004c62ea0}, 0xc00c4f2d20?, {0xc03be7aea0, {0x17600a7688712da3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc010d166f0 sp=0xc010d165b0 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc004c62ea0}, 0xc0030a8690, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc010d16a10 sp=0xc010d166f0 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a7688712da3, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc010d16dd8 sp=0xc010d16a10 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a7688712da3, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc010d17760 sp=0xc010d16dd8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x20?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc010d179a8 sp=0xc010d17760 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc004c62e10}, {0x0?}, 0xc0030a83c0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc010d18018 sp=0xc010d179a8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc004c62d20}, 0xc0030a83c0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc010d180c0 sp=0xc010d18018 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc004c62d20?}, {0x52f0080?, 0xc0030a83c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc010d180f0 sp=0xc010d180c0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc004c62d20}, {0x52f0080, 0xc0030a83c0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc010d18420 sp=0xc010d180f0 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc004c62d20?}, {0x52f0080?, 0xc0030a83c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010d18460 sp=0xc010d18420 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc004c62d20}, {0x52f0080, 0xc0030a83c0}, 0xc010d184f8?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc010d18490 sp=0xc010d18460 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc004c62d20?}, {0x52f0080?, 0xc0030a83c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010d184d0 sp=0xc010d18490 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc004c62d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc010d18508 sp=0xc010d184d0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc004c62d20}, {0xc001821c20?, 0xc01572e840?}, 0xc010d185c0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc010d18580 sp=0xc010d18508 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc004c62d20?}, {0x52f0080?, 0xc0030a83c0?}, 0xc03b681ea0?, 0x4b67c20?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc010d185f8 sp=0xc010d18580 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc004c62d20?}, {0x52f0080?, 0xc0030a83c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010d18638 sp=0xc010d185f8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc004c62d20?}, {0xc004c62d20?, 0x4?}, {0x52f0080?, 0xc0030a83c0?}, {0x51b9a20?, 0xc01958e880}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc010d18678 sp=0xc010d18638 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc004c62d20}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0030a83c0}, {0x51b9a20, 0xc01958e880}, 0xc010d187f0?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc010d18768 sp=0xc010d18678 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc004c62d20}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0030a83c0}, {0x51b9a20, 0xc01958e880}, 0xc010d188d0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc010d18800 sp=0xc010d18768 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc004c62ae0}, 0xc0030a83c0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc010d18880 sp=0xc010d18800 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xcd1e20?, {0x624a948?, 0xc004c62ae0?}, 0xc010d18960?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc010d18908 sp=0xc010d18880 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc013666360, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010d18ba8 sp=0xc010d18908 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc013666360, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010d18ce0 sp=0xc010d18ba8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010d19428 sp=0xc010d18ce0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010d19d28 sp=0xc010d19428 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc010d1a590 sp=0xc010d19d28 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76889bd78a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc010d1a9b0 sp=0xc010d1a590 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc017453038, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc010d1ac08 sp=0xc010d1a9b0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc017453000, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc010d1ad30 sp=0xc010d1ac08 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc017452fd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc010d1afc0 sp=0xc010d1ad30 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc017452ed0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc010d1b560 sp=0xc010d1afc0 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc017452ed0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc010d1b850 sp=0xc010d1b560 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc017452da0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc010d1bb50 sp=0xc010d1b850 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc017452d80?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc010d1bcf8 sp=0xc010d1bb50 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc017452cd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc010d1bf10 sp=0xc010d1bcf8 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc017452b00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc010d1c168 sp=0xc010d1bf10 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc010d1c290 sp=0xc010d1c168 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc02b24d1e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc010d1c558 sp=0xc010d1c290 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc010d1c670 sp=0xc010d1c558 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00a7f3200, {0x624a948, 0xc004c62780}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc010d1c890 sp=0xc010d1c670 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00a7f3200, {0x624a948, 0xc004c62780}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc010d1cb88 sp=0xc010d1c890 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc013dedbc0, {0x624a948, 0xc004c62780}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc010d1cc68 sp=0xc010d1cb88 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00a358e00, {0x624a948, 0xc004c62780}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc010d1cf40 sp=0xc010d1cc68 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc01fd49ae0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc010d1cf98 sp=0xc010d1cf40 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc005433140) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc010d1cfb0 sp=0xc010d1cf98 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc00448bbd0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:135 +0x2e fp=0xc010d1cff8 sp=0xc010d1cfb0 pc=0x2860a0e github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecprojconst.projPlusInt64Int64ConstOp.Next({{{{{...}}, {{...}}}, 0xc010dc7bf0, 0xa, 0xb, 0x0}, 0x1}) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/colexecprojconst/proj_const_right_ops.eg.go:4023 +0x87 fp=0xc010d1d120 sp=0xc010d1cff8 pc=0x2ea3c27 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecprojconst.(*projPlusInt64Int64ConstOp).Next(0xc005275100?) :1 +0xa5 fp=0xc010d1d1c0 sp=0xc010d1d120 pc=0x2ff39e5 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc001821c20) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc010d1d1e8 sp=0xc010d1d1c0 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc001821c20) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc010d1d200 sp=0xc010d1d1e8 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc010d1d218 sp=0xc010d1d200 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x62841c0?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc010d1d258 sp=0xc010d1d218 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc001821c20) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc010d1d290 sp=0xc010d1d258 pc=0x29a100c created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 3964 [runnable]: runtime.gopark(0x0?, 0xb?, 0x0?, 0x0?, 0x9ca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b2caa0 sp=0xc001b2ca80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x534825?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc001b2cad8 sp=0xc001b2caa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbcf88, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc001b2caf8 sp=0xc001b2cad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00418e000?, 0xc004d22000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001b2cb20 sp=0xc001b2caf8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00418e000, {0xc004d22000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc001b2cba0 sp=0xc001b2cb20 pc=0x535efa net.(*netFD).Read(0xc00418e000, {0xc004d22000?, 0x459e71?, 0x4d964e0?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc001b2cbe8 sp=0xc001b2cba0 pc=0x7a20c9 net.(*conn).Read(0xc002e01ae0, {0xc004d22000?, 0x173?, 0xc001b2cc98?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc001b2cc30 sp=0xc001b2cbe8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0040e2180, {0xc004d22000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc001b2cc78 sp=0xc001b2cc30 pc=0x1641f96 bufio.(*Reader).Read(0xc004c902a0, {0xc004a1d460, 0x9, 0x100?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc001b2ccb0 sp=0xc001b2cc78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc004c902a0}, {0xc004a1d460, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc001b2ccf8 sp=0xc001b2ccb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004a1d460?, 0x9?, 0x0?}, {0x62052c0?, 0xc004c902a0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc001b2cd48 sp=0xc001b2ccf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc004a1d420) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc001b2cdf8 sp=0xc001b2cd48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc004caa680, 0xc001b2cfd0?, 0x1054c46?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc001b2cf18 sp=0xc001b2cdf8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc004caa680}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc001b2cf80 sp=0xc001b2cf18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc001b2cfe0 sp=0xc001b2cf80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b2cfe8 sp=0xc001b2cfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 3732 [select, 9 minutes]: runtime.gopark(0xc001850fb0?, 0x2?, 0x6a?, 0x56?, 0xc001850f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001850e28 sp=0xc001850e08 pc=0x4907d6 runtime.selectgo(0xc001850fb0, 0xc001850f98, 0xc002a40360?, 0x0, 0xc006e85410?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001850f68 sp=0xc001850e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc001850fe0 sp=0xc001850f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001850fe8 sp=0xc001850fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 3803 [select, 9 minutes]: runtime.gopark(0xc00143ffb0?, 0x2?, 0x20?, 0xfb?, 0xc00143ff94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00143fe18 sp=0xc00143fdf8 pc=0x4907d6 runtime.selectgo(0xc00143ffb0, 0xc00143ff90, 0xc0037f2700?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00143ff58 sp=0xc00143fe18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc00143ffe0 sp=0xc00143ff58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00143ffe8 sp=0xc00143ffe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 423287 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x243?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0103e8f58 sp=0xc0103e8f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539450?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0103e8f90 sp=0xc0103e8f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539368, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0103e8fb0 sp=0xc0103e8f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc03cf18f80?, 0xc00d615000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0103e8fd8 sp=0xc0103e8fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc03cf18f80, {0xc00d615000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0103e9058 sp=0xc0103e8fd8 pc=0x535efa net.(*netFD).Read(0xc03cf18f80, {0xc00d615000?, 0x7b5a68?, 0xc03cf18f80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0103e90a0 sp=0xc0103e9058 pc=0x7a20c9 net.(*conn).Read(0xc013f342f0, {0xc00d615000?, 0x164290f?, 0xc013f342f0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0103e90e8 sp=0xc0103e90a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc010c2b920, {0xc00d615000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc0103e9130 sp=0xc0103e90e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc012af25d0, {0xc00d615000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc0103e91d0 sp=0xc0103e9130 pc=0x370f896 bufio.(*Reader).fill(0xc00d3ae090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc0103e9208 sp=0xc0103e91d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00d3ae090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc0103e9228 sp=0xc0103e9208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00d3ae668?, {0x62295c8, 0xc00d3ae090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc0103e9260 sp=0xc0103e9228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00d3ae000, 0xc00ae1d0c0, 0xc0103e9442, 0xc0103e9444, {0x623b718, 0xc0067a6f30}, 0xc0059fa000, 0xc013ababa4, 0xc0103e9440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc0103e9398 sp=0xc0103e9260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00d3ae000, {0x624a948, 0xc03f27eb40}, 0xc00ae1d070, 0xc0059fa000, 0xc03f27ea80, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc0103e9768 sp=0xc0103e9398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc03f27eb40}, {0x628a538, 0xc010c2b920}, {{{0xc004804038, 0x4}}, 0x1, 0xc03f27eab0, 0xc03f27eae0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc0103e98a8 sp=0xc0103e9768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc03f27eb40}, {0x628a538?, 0xc010c2b920}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc0103e9e20 sp=0xc0103e98a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc03f27e900?}, {0x628a538, 0xc010c2b920}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc0103e9eb0 sp=0xc0103e9e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc03f27e900}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc0103e9f30 sp=0xc0103e9eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0103e9fe0 sp=0xc0103e9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0103e9fe8 sp=0xc0103e9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3731 [chan receive, 9 minutes]: runtime.gopark(0xc006ae0460?, 0x624a948?, 0xe0?, 0x79?, 0xc000920000?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002733558 sp=0xc002733538 pc=0x4907d6 runtime.chanrecv(0xc001bfe780, 0xc0027336f8, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0027335e8 sp=0xc002733558 pc=0x45a91b runtime.chanrecv1(0x176009f82153225c?, 0x0?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc002733610 sp=0xc0027335e8 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc006ad8800, 0x117?, 0xc0043b5560, {0x7f275f2bd8e0, 0xc00090a010}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:250 +0x707 fp=0xc0027337f0 sp=0xc002733610 pc=0x1e00027 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:152 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00679ea80, 0xc0043b5560, {0x7f275f2bd8e0, 0xc00090a010}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3253 +0x10c fp=0xc002733850 sp=0xc0027337f0 pc=0x1e3af0c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc004469900?, 0xc0043b5560, {0x7f275f2bd8e0, 0xc00090a010}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:228 +0xf6 fp=0xc0027338b8 sp=0xc002733850 pc=0x1e64156 github.com/cockroachdb/cockroach/pkg/server.(*Node).singleRangeFeed(0x45efeb?, 0x50c16c0?, {0x7f275f2bd8e0?, 0xc00090a010?}) github.com/cockroachdb/cockroach/pkg/server/node.go:1341 +0x38 fp=0xc002733910 sp=0xc0027338b8 pc=0x39c1118 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x50c16c0?, 0x0?, {0x62893f0?, 0xc00090a010}) github.com/cockroachdb/cockroach/pkg/server/node.go:1335 +0x4e fp=0xc002733950 sp=0xc002733910 pc=0x39c108e github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x624a948?}, {0x627d570?, 0xc004469900?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:980 +0x72 fp=0xc0027339a0 sp=0xc002733950 pc=0x165b012 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x52b4780, 0xc00a1cf500}, {0x6279ce8?, 0xc002ba0de0?}, 0x8fb69e0, 0xc001bfe720) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc002733cd8 sp=0xc0027339a0 pc=0x16481a4 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x52b4780?, 0xc00a1cf500?}, {0x6279ce8?, 0xc002ba0de0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:827 +0x5b fp=0xc002733d18 sp=0xc002733cd8 pc=0x165a09b github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x52b4780, 0xc00a1cf500}, {0x6279ce8, 0xc002ba0de0}, 0x1054f90?, 0xc001e2f900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc002733d48 sp=0xc002733d18 pc=0x16576d4 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x52b4780?, 0xc00a1cf500?}, {0x6279ce8?, 0xc002ba0de0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:830 +0x83 fp=0xc002733d88 sp=0xc002733d48 pc=0x165a0c3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0xc003151200?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc002733db8 sp=0xc002733d88 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc002ba0db0}, {0xc001439ec0?, 0x46159f?}, 0xc001439e78) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc002733e30 sp=0xc002733db8 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x52b4780, 0xc00a1cf500}, {0x6279ce8?, 0xc002ba0de0?}, 0x8fb69e0, 0xc001e2f900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc002733eb8 sp=0xc002733e30 pc=0x165790f github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x52b4780?, 0xc00a1cf500?}, {0x6279ce8?, 0xc002ba0de0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:830 +0x83 fp=0xc002733ef8 sp=0xc002733eb8 pc=0x165a0c3 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc001a44c20, 0x3, 0x4}, {0x52b4780, 0xc00a1cf500}, {0x6279ce8, 0xc002ba0de0}, 0x8fb69e0, 0xc001bfe720) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:832 +0x142 fp=0xc002733f30 sp=0xc002733ef8 pc=0x1659fa2 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:985 +0x117 fp=0xc002733fe0 sp=0xc002733f30 pc=0x165af17 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002733fe8 sp=0xc002733fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:973 +0x2dd goroutine 493366 [sleep]: runtime.gopark(0x4b8709ec692?, 0x220998?, 0xaa?, 0x16?, 0x802c560f07?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079a4df0 sp=0xc0079a4dd0 pc=0x4907d6 time.Sleep(0x117c9b686) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc0079a4e30 sp=0xc0079a4df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc003522c30, 0x220998, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc0079a4e88 sp=0xc0079a4e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc003522c30, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc0079a4ec8 sp=0xc0079a4e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00c9fdf40?, 0x1, 0xc00885ff70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc0079a4fa8 sp=0xc0079a4ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc0079a4fe0 sp=0xc0079a4fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079a4fe8 sp=0xc0079a4fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 1291 [select]: runtime.gopark(0xc017511ea8?, 0x2?, 0x0?, 0x0?, 0xc017511e3c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c820ca0 sp=0xc00c820c80 pc=0x4907d6 runtime.selectgo(0xc00c820ea8, 0xc017511e38, 0x6207560?, 0x0, 0x17?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c820de0 sp=0xc00c820ca0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x624a948, 0xc0016e3d10}) github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:308 +0x330 fp=0xc00c820f30 sp=0xc00c820de0 pc=0x373bcf0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c820fe0 sp=0xc00c820f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c820fe8 sp=0xc00c820fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1292 [select]: runtime.gopark(0xc00a96dea8?, 0x2?, 0x0?, 0x0?, 0xc00a96de3c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a96dca0 sp=0xc00a96dc80 pc=0x4907d6 runtime.selectgo(0xc00a96dea8, 0xc00a96de38, 0x6207560?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a96dde0 sp=0xc00a96dca0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x624a948, 0xc0016e3d40}) github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:308 +0x330 fp=0xc00a96df30 sp=0xc00a96dde0 pc=0x373bcf0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a96dfe0 sp=0xc00a96df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a96dfe8 sp=0xc00a96dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1293 [select]: runtime.gopark(0xc00a2c5ea8?, 0x2?, 0x0?, 0x0?, 0xc00a2c5e3c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00697bca0 sp=0xc00697bc80 pc=0x4907d6 runtime.selectgo(0xc00697bea8, 0xc00a2c5e38, 0x6207580?, 0x0, 0x23?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00697bde0 sp=0xc00697bca0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x624a948, 0xc0016e3d70}) github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:308 +0x330 fp=0xc00697bf30 sp=0xc00697bde0 pc=0x373bcf0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00697bfe0 sp=0xc00697bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00697bfe8 sp=0xc00697bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498407 [select]: runtime.gopark(0xc012c07208?, 0x2?, 0x98?, 0xad?, 0xc012c07204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012c07090 sp=0xc012c07070 pc=0x4907d6 runtime.selectgo(0xc012c07208, 0xc012c07200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc012c071d0 sp=0xc012c07090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00c8b0120) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc012c07238 sp=0xc012c071d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ef82420, {0x51b9a20?, 0xc00418ea00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc012c07340 sp=0xc012c07238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc012c07370 sp=0xc012c07340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc003563d40, 0xc012c07448, 0xc012c07418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc012c073c8 sp=0xc012c07370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc003563d40, {0x51b9a20?, 0xc00418ea00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc012c07478 sp=0xc012c073c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc019593110?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc01068f1d0}, {0x51b9a20, 0xc00418ea00}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc012c074e0 sp=0xc012c07478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc019593110}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc01068f1d0}, {0x51b9a20, 0xc00418ea00}, 0x33b411acb29866a5?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc012c075d0 sp=0xc012c074e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc019593110?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc01068f1d0?}, {0x51b9a20?, 0xc00418ea00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc012c07670 sp=0xc012c075d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e3b38, {0x624a948, 0xc019593110}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc012c076f0 sp=0xc012c07670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e3b38?}}, {0x624a948, 0xc019593110}, 0xc01068f1d0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc012c077c8 sp=0xc012c076f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xb0fb830?, {0x624a948?, 0xc019593110?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00b0fb928?}) :1 +0x65 fp=0xc012c07818 sp=0xc012c077c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc003d2d920, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc012c07ab8 sp=0xc012c07818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc003d2d920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e13887b9, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc012c07bf0 sp=0xc012c07ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e13887b9, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc012c08338 sp=0xc012c07bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e13887b9, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc012c08c38 sp=0xc012c08338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e13887b9, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc012c094a0 sp=0xc012c08c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e13887b9, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc012c098c0 sp=0xc012c094a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc012c099f8 sp=0xc012c098c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc0017083c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc012c09c00 sp=0xc012c099f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc0017083c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc012c09f30 sp=0xc012c09c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012c09fe0 sp=0xc012c09f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012c09fe8 sp=0xc012c09fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498448 [select]: runtime.gopark(0xc00a5ffc20?, 0x3?, 0x22?, 0xfb?, 0xc00a5ffbda?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a5ffa40 sp=0xc00a5ffa20 pc=0x4907d6 runtime.selectgo(0xc00a5ffc20, 0xc00a5ffbd4, 0xc012546060?, 0x0, 0x1c011e980a0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a5ffb80 sp=0xc00a5ffa40 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc005b2af20, {0x624a910, 0xc012546060}, 0xc00a5ffca8?, {0x629caa8, 0xc003165c80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x165 fp=0xc00a5ffc60 sp=0xc00a5ffb80 pc=0x1ca0605 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0005593b0, {0x624a910, 0xc012546060}, 0xc00a5ffdd0?, 0xc0057a67e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:749 +0x3d1 fp=0xc00a5ffda8 sp=0xc00a5ffc60 pc=0x1ca75d1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x624a910?, 0xc012546060?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:792 +0x38 fp=0xc00a5ffde0 sp=0xc00a5ffda8 pc=0x1ca7bd8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x624a948?, 0xc020409e30?}, {0x537e81f, 0x12}, 0x4a817c800, 0xc00a5fff00) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed fp=0xc00a5ffe80 sp=0xc00a5ffde0 pc=0x16210cd github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x624a948, 0xc020409e30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:790 +0xac fp=0xc00a5fff30 sp=0xc00a5ffe80 pc=0x1ca7aac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a5fffe0 sp=0xc00a5fff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a5fffe8 sp=0xc00a5fffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 1297 [semacquire, 9 minutes]: runtime.gopark(0x18?, 0xc000580400?, 0x40?, 0xb1?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002f4daa8 sp=0xc002f4da88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc001cfe650, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc002f4db10 sp=0xc002f4daa8 pc=0x4a17de sync.runtime_Semacquire(0xc001cfe640?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc002f4db40 sp=0xc002f4db10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc002f4db68 sp=0xc002f4db40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc001cfe640) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc002f4db80 sp=0xc002f4db68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc001cfe640?, {0x624a8a0?, 0xc001cfe600?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc002f4dbb0 sp=0xc002f4db80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc001ad9c40}, {0xc007e98900, 0x2, 0xc0032b2dd0?}, 0x0, 0xc0019d0b40, {0xc000029650, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc002f4dd38 sp=0xc002f4dbb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x32fa9a5?, {0x624a8a0, 0xc001ad9c40}, {0xc004959bc0, 0x2, 0xc448e6?}, {0x626b400?, 0x32b2f00?, 0xc0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc002f4de58 sp=0xc002f4dd38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc0016e3d40?, {0x624a8a0?, 0xc001ad9c40?}, {0xc004959bc0?, 0xc0032b2ee8?, 0xc0032b2fd0?}, {0x1054c46?, 0x624a948?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc002f4ded0 sp=0xc002f4de58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc001ad9c40?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc002f4df58 sp=0xc002f4ded0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc002f4df78 sp=0xc002f4df58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc002f4dfe0 sp=0xc002f4df78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002f4dfe8 sp=0xc002f4dfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 3490 [select]: runtime.gopark(0xc00a3d3e18?, 0x2?, 0xa0?, 0xb6?, 0xc00a3d3dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a3d3c68 sp=0xc00a3d3c48 pc=0x4907d6 runtime.selectgo(0xc00a3d3e18, 0xc00a3d3df8, 0xc0009ca780?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a3d3da8 sp=0xc00a3d3c68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001786000, {0x624a8a0, 0xc001ad9c40}, 0xc0016e3d10?, 0xc0019d0b40) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc00a3d3f20 sp=0xc00a3d3da8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc001ad9c40?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc00a3d3f58 sp=0xc00a3d3f20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00a3d3f78 sp=0xc00a3d3f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00a3d3fe0 sp=0xc00a3d3f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a3d3fe8 sp=0xc00a3d3fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498226 [runnable]: runtime.gopark(0xae1b627e591?, 0xae1861de057?, 0xe0?, 0x1c?, 0xc0152b4d80?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a74c380 sp=0xc00a74c360 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00a74c3e8 sp=0xc00a74c380 pc=0x4a17de sync.runtime_SemacquireMutex(0xc0005ee3e0?, 0x78?, 0x46b793?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00a74c418 sp=0xc00a74c3e8 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x46b0eb?, 0x7?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00a74c470 sp=0xc00a74c418 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x9c624c0?, 0x203003?, 0xc00c730900?, 0x3c?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00a74c4b8 sp=0xc00a74c470 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004e6b340, {0xc00a74c5f8?, 0x4ccece?}, 0x0, 0x0, 0xc014072310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00a74c5b8 sp=0xc00a74c4b8 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00222e300, 0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc00a74c608 sp=0xc00a74c5b8 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc014072310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc00a74c690 sp=0xc00a74c608 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc002941970, 0x2f68ab0?, 0xc014073240, {0x0?, 0xc014072310?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc00a74c7c0 sp=0xc00a74c690 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x7f25833d1944?, 0x0?, 0xc007495540?, {0x0?, 0xc014072310?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc00a74c810 sp=0xc00a74c7c0 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc0140731f0?, 0xc005abd080?, {0x0?, 0xc014072310?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc00a74c858 sp=0xc00a74c810 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0140731f0, 0xc005abd080, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc00a74c8b8 sp=0xc00a74c858 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0140731f0, {0xc007495540, 0x21, 0x40}, {0xc015b02300, 0x21, 0x21}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc00a74c908 sp=0xc00a74c8b8 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc014072520, {0xc015b02300?, 0x0?, 0x0?}, 0x0?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00a74ca38 sp=0xc00a74c908 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc014072520, {0xc007495540?, 0x0?, 0x0?}, {0xc015b02300?, 0x0?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00a74ca78 sp=0xc00a74ca38 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc014072388, {0xc007495540?, 0x0?, 0x21?}, {0xc015b02300, 0x21, 0x21}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00a74cad0 sp=0xc00a74ca78 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc014072000, {0xc015b02300, 0x21, 0x21}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00a74cb68 sp=0xc00a74cad0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0076f0588, {{0xc00a4e9ec0, 0x20, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00a74cbb8 sp=0xc00a74cb68 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00b535b00, {0x624a948, 0xc03f1f73b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc00a74cc20 sp=0xc00a74cbb8 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc03f1f73b0}, {0x62b4fe0, 0xc0076f0588}, {0xc00a4e9ec0, 0x20, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc00a74d1f0 sp=0xc00a74cc20 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc03f1f73b0}, {0x628f760?, 0xc0076f0580?}, {0xc00a4e9ec0, 0x20, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc00a74d370 sp=0xc00a74d1f0 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc03f1f73b0?}, {0x628f760?, 0xc0076f0580?}, {0xc00a4e9ec0?, 0x34?, 0x14?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc00a74d460 sp=0xc00a74d370 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc0167177a0}, {{0x17600a76e1925df7, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_txn.go:71 +0x317 fp=0xc00a74d8a8 sp=0xc00a74d460 pc=0x1cfd357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e1925df7, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc00a74e138 sp=0xc00a74d8a8 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00a60f1d0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00a74ffa8 sp=0xc00a74e138 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00a750a18, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc00a750618 sp=0xc00a74ffa8 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc00130a400, {0x624a948, 0xc03f1f73b0}, 0xc00a60f1d0, 0xc003025720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc00a7512a8 sp=0xc00a750618 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00130a400, {0x624a948, 0xc03f1f73b0}, 0xc00a60f1d0, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00a7515c8 sp=0xc00a7512a8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00130a400, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00a751990 sp=0xc00a7515c8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00a752318 sp=0xc00a751990 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00a752560 sp=0xc00a752318 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc03f1f7350}, {0x7f27639619e0?}, 0xc00a60f0e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00a752bd0 sp=0xc00a752560 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc03f1f72c0}, 0xc00a60f0e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00a752c78 sp=0xc00a752bd0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc03f1f72c0?}, {0x52f0080?, 0xc00a60f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00a752ca8 sp=0xc00a752c78 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc03f1f72c0}, {0x52f0080, 0xc00a60f0e0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00a752fd8 sp=0xc00a752ca8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03f1f72c0?}, {0x52f0080?, 0xc00a60f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a753018 sp=0xc00a752fd8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc03f1f72c0}, {0x52f0080, 0xc00a60f0e0}, 0xc00a7530b0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00a753048 sp=0xc00a753018 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03f1f72c0?}, {0x52f0080?, 0xc00a60f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a753088 sp=0xc00a753048 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc03f1f72c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00a7530c0 sp=0xc00a753088 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc03f1f72c0}, {0xc006b6de00?, 0x0?}, 0xc00a753178) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00a753138 sp=0xc00a7530c0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc03f1f72c0?}, {0x52f0080?, 0xc00a60f0e0?}, 0xc00b0fb100?, 0x0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00a7531b0 sp=0xc00a753138 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03f1f72c0?}, {0x52f0080?, 0xc00a60f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a7531f0 sp=0xc00a7531b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc03f1f72c0?}, {0xc03f1f72c0?, 0x4?}, {0x52f0080?, 0xc00a60f0e0?}, {0x51b9a20?, 0xc00418ff80}, 0x20300f?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00a753230 sp=0xc00a7531f0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03f1f72c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a60f0e0}, {0x51b9a20, 0xc00418ff80}, 0xc00a7533a8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00a753320 sp=0xc00a753230 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc03f1f72c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a60f0e0}, {0x51b9a20, 0xc00418ff80}, 0xc00a753488?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00a7533b8 sp=0xc00a753320 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc03f1f7260}, 0xc00a60f0e0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00a753438 sp=0xc00a7533b8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc00b0fb4d8?, {0x624a948?, 0xc03f1f7260?}, 0xc00b0fb538?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00a7534c0 sp=0xc00a753438 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc03d859e60, {_, _}, _, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00a753760 sp=0xc00a7534c0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc03d859e60, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00a753898 sp=0xc00a753760 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00a753fe0 sp=0xc00a753898 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00a7548e0 sp=0xc00a753fe0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00a755148 sp=0xc00a7548e0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x17600a76e1926d39, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00a755568 sp=0xc00a755148 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00a7556a0 sp=0xc00a755568 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00a7557c8 sp=0xc00a7556a0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0x4b6f420?, {_, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc00a7558e8 sp=0xc00a7557c8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e1925df7, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00a7559f8 sp=0xc00a7558e8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc01502a0f0}, 0x0?, 0xc01ad54b00) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00a755bf0 sp=0xc00a7559f8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc01502a0f0}, 0x4596bd?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc00a755c38 sp=0xc00a755bf0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc00189b7a0, {0x624a948, 0xc01502a0f0}, {{0xe8, 0x53, 0x5, 0x8c, 0x29, 0xd7, 0x44, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc00a755c88 sp=0xc00a755c38 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).startQueryPusherTxn.func1({0x624a948, 0xc01502a0f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:854 +0x238 fp=0xc00a755f30 sp=0xc00a755c88 pc=0x19bbe78 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a755fe0 sp=0xc00a755f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a755fe8 sp=0xc00a755fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3493 [select, 9 minutes]: runtime.gopark(0xc0032b2e58?, 0x2?, 0x0?, 0x30?, 0xc0032b2dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b2c80 sp=0xc0032b2c60 pc=0x4907d6 runtime.selectgo(0xc0032b2e58, 0xc0032b2df8, 0xc001cfe600?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b2dc0 sp=0xc0032b2c80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc001cfe600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc0032b2f58 sp=0xc0032b2dc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc0032b2f78 sp=0xc0032b2f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc0032b2fe0 sp=0xc0032b2f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b2fe8 sp=0xc0032b2fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 3714 [select]: runtime.gopark(0xc008d84d68?, 0x2?, 0x5?, 0x0?, 0xc008d84d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008d84bb8 sp=0xc008d84b98 pc=0x4907d6 runtime.selectgo(0xc008d84d68, 0xc008d84d30, 0x8?, 0x0, 0xc0231de000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008d84cf8 sp=0xc008d84bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00491eb40, {0xc001f7e7f0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc008d84d98 sp=0xc008d84cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00491eb40, {0xc001f7e7f0?, 0xc0072f98f0?, 0xc008d84de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc008d84dd8 sp=0xc008d84d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002b392c0, {0xc001f7e7f0?, 0xc008d84e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc008d84e20 sp=0xc008d84dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc002b392c0}, {0xc001f7e7f0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc008d84e68 sp=0xc008d84e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0048d3560, {0xc001f7e7f0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc008d84ea8 sp=0xc008d84e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc001f7e7e0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc008d84f28 sp=0xc008d84ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc007bd9028?, 0xc0048d3560, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc008d84fc0 sp=0xc008d84f28 pc=0xc19546 google.golang.org/grpc.recv(0x6?, {0x7f275f301038, 0xc0000af170}, 0xc2b369?, {0x0?, 0x0?}, {0x51d3160, 0xc00e1c4fc0}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc008d85038 sp=0xc008d84fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc004e33e40, {0x51d3160?, 0xc00e1c4fc0}, 0x5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc008d85140 sp=0xc008d85038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc008d85170 sp=0xc008d85140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0048d3200, 0xc008d85248, 0xc008d85218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc008d851c8 sp=0xc008d85170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0048d3200, {0x51d3160?, 0xc00e1c4fc0?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc008d85278 sp=0xc008d851c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0039e7740, {0x51d3160?, 0xc00e1c4fc0?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc008d852c0 sp=0xc008d85278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc0008d9c70) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc008d852f0 sp=0xc008d852c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc008d85310 sp=0xc008d852f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00491e910, 0xc008d85780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc008d853b8 sp=0xc008d85310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc001cfe600?}, {{0xc0035cdb68, 0x1, 0x8}, {0xc0035cdb70, 0x1, 0x8}}, {0x176009f829266a9b, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc008d859a0 sp=0xc008d853b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc001cfe600}, 0xc004731d10, 0xc0035cdb68?, {{0xc0035cdb68, 0x1, 0x8}, {0xc0035cdb70, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc008d85e80 sp=0xc008d859a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc001cfe600?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc008d85f58 sp=0xc008d85e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc008d85f78 sp=0xc008d85f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc008d85fe0 sp=0xc008d85f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008d85fe8 sp=0xc008d85fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 470320 [sleep]: runtime.gopark(0x4b935cb15e0?, 0x400ea1?, 0x46?, 0x7d?, 0x811ebc9d98?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e19fdf0 sp=0xc00e19fdd0 pc=0x4907d6 time.Sleep(0x14503efea) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00e19fe30 sp=0xc00e19fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00d10bb00, 0x400ea1, {0x4618e7?, 0x203003?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00e19fe88 sp=0xc00e19fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00d10bb00, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00e19fec8 sp=0xc00e19fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00c9de280?, 0xc, 0xc003eacf70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00e19ffa8 sp=0xc00e19fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00e19ffe0 sp=0xc00e19ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e19ffe8 sp=0xc00e19ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 489577 [sleep]: runtime.gopark(0x4b870f13ba6?, 0x801922?, 0x8b?, 0x7e?, 0x80301076e8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b04df0 sp=0xc005b04dd0 pc=0x4907d6 time.Sleep(0x11b31a459) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc005b04e30 sp=0xc005b04df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0054611a0, 0x801922, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc005b04e88 sp=0xc005b04e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0054611a0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc005b04ec8 sp=0xc005b04e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc0057d5900?, {0xc03ba67360?, 0x2, 0xc026e0fda0?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc005b04fa8 sp=0xc005b04ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc005b04fe0 sp=0xc005b04fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b04fe8 sp=0xc005b04fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 380326 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x148?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a264f58 sp=0xc00a264f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539090?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00a264f90 sp=0xc00a264f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764538fa8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00a264fb0 sp=0xc00a264f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc01956f600?, 0xc01282a000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00a264fd8 sp=0xc00a264fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01956f600, {0xc01282a000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00a265058 sp=0xc00a264fd8 pc=0x535efa net.(*netFD).Read(0xc01956f600, {0xc01282a000?, 0x7b5a68?, 0xc01956f600?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00a2650a0 sp=0xc00a265058 pc=0x7a20c9 net.(*conn).Read(0xc007cabb98, {0xc01282a000?, 0x164290f?, 0xc007cabb98?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00a2650e8 sp=0xc00a2650a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc007ec9260, {0xc01282a000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00a265130 sp=0xc00a2650e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc005bc1350, {0xc01282a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00a2651d0 sp=0xc00a265130 pc=0x370f896 bufio.(*Reader).fill(0xc003490490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00a265208 sp=0xc00a2651d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003490490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00a265228 sp=0xc00a265208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003490a68?, {0x62295c8, 0xc003490490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00a265260 sp=0xc00a265228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003490400, 0xc01607dd40, 0xc00a265442, 0xc00a265444, {0x623b718, 0xc0304fbef0}, 0xc0059fa000, 0xc0143aa024, 0xc00a265440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00a265398 sp=0xc00a265260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003490400, {0x624a948, 0xc00e2177d0}, 0xc01607dd30, 0xc0059fa000, 0xc00e217710, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00a265768 sp=0xc00a265398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc00e2177d0}, {0x628a538, 0xc007ec9260}, {{{0xc002a6e038, 0x4}}, 0x1, 0xc00e217740, 0xc00e217770, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00a2658a8 sp=0xc00a265768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc00e2177d0}, {0x628a538?, 0xc007ec9260}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00a265e20 sp=0xc00a2658a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00e217650?}, {0x628a538, 0xc007ec9260}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00a265eb0 sp=0xc00a265e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00e217650}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00a265f30 sp=0xc00a265eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a265fe0 sp=0xc00a265f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a265fe8 sp=0xc00a265fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3505 [select, 9 minutes]: runtime.gopark(0xc0032b57b0?, 0x2?, 0x50?, 0x56?, 0xc0032b579c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b5628 sp=0xc0032b5608 pc=0x4907d6 runtime.selectgo(0xc0032b57b0, 0xc0032b5798, 0x6221ba0?, 0x0, 0x624a948?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b5768 sp=0xc0032b5628 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0032b57e0 sp=0xc0032b5768 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b57e8 sp=0xc0032b57e0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 3715 [select, 9 minutes]: runtime.gopark(0xc0032b67b0?, 0x2?, 0x68?, 0xdb?, 0xc0032b6794?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b6618 sp=0xc0032b65f8 pc=0x4907d6 runtime.selectgo(0xc0032b67b0, 0xc0032b6790, 0xc0032b67d0?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b6758 sp=0xc0032b6618 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0032b67e0 sp=0xc0032b6758 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b67e8 sp=0xc0032b67e0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 3716 [select, 9 minutes]: runtime.gopark(0xc0032b4fb0?, 0x2?, 0xb0?, 0x5c?, 0xc0032b4f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b4e28 sp=0xc0032b4e08 pc=0x4907d6 runtime.selectgo(0xc0032b4fb0, 0xc0032b4f98, 0xc002a40360?, 0x0, 0xc00325c9f0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b4f68 sp=0xc0032b4e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0032b4fe0 sp=0xc0032b4f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b4fe8 sp=0xc0032b4fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 498080 [select]: runtime.gopark(0xc00c32be60?, 0x3?, 0x0?, 0x0?, 0xc00c32be32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c32bcb8 sp=0xc00c32bc98 pc=0x4907d6 runtime.selectgo(0xc00c32be60, 0xc00c32be2c, 0xc0053fd740?, 0x0, 0xc005976e18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c32bdf8 sp=0xc00c32bcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc00d1d02d0, {0x624a948, 0xc0053fd740}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00c32beb0 sp=0xc00c32bdf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc0053fd740?}) :1 +0x39 fp=0xc00c32bed8 sp=0xc00c32beb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc0053fd740}, {0x53ddbcd?, 0x6294e00?}, 0xc007cf1fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00c32bf50 sp=0xc00c32bed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00c32bfe0 sp=0xc00c32bf50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c32bfe8 sp=0xc00c32bfe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 498420 [select]: runtime.gopark(0xc00a233300?, 0x3?, 0x4?, 0x0?, 0xc00a232fca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a232d50 sp=0xc00a232d30 pc=0x4907d6 runtime.selectgo(0xc00a233300, 0xc00a232fc4, 0x17600a76a34b5975?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a232e90 sp=0xc00a232d50 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc0134d2cc0}, 0xc00a5af860, 0xc00185c500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00a233600 sp=0xc00a232e90 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc0134d2cc0}, 0xc00a5af860, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00a233920 sp=0xc00a233600 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76a34b5975, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00a233ce8 sp=0xc00a233920 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76a34b5975, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00a234670 sp=0xc00a233ce8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x1?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00a2348b8 sp=0xc00a234670 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0134d2c60}, {0x629d9a8?}, 0xc00a5af770) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00a234f28 sp=0xc00a2348b8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0134d2ba0}, 0xc00a5af770) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00a234fd0 sp=0xc00a234f28 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc0134d2ba0?}, {0x52f0080?, 0xc00a5af770?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00a235000 sp=0xc00a234fd0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0134d2ba0}, {0x52f0080, 0xc00a5af770}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00a235330 sp=0xc00a235000 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc0134d2ba0?}, {0x52f0080?, 0xc00a5af770?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a235370 sp=0xc00a235330 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0134d2ba0}, {0x52f0080, 0xc00a5af770}, 0xc00d1ad408?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00a2353a0 sp=0xc00a235370 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc0134d2ba0?}, {0x52f0080?, 0xc00a5af770?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a2353e0 sp=0xc00a2353a0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0134d2ba0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00a235418 sp=0xc00a2353e0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0134d2ba0}, {0x7f275fc94a58?, 0x0?}, 0xc00d1ad4d0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00a235490 sp=0xc00a235418 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0134d2ba0?}, {0x52f0080?, 0xc00a5af770?}, 0xff3d15f6d74568b4?, 0x1?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00a235508 sp=0xc00a235490 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc0134d2ba0?}, {0x52f0080?, 0xc00a5af770?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00a235548 sp=0xc00a235508 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc0134d2ba0?}, {0xc0134d2ba0?, 0x4?}, {0x52f0080?, 0xc00a5af770?}, {0x51b9a20?, 0xc010b7b680}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00a235588 sp=0xc00a235548 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0134d2ba0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a5af770}, {0x51b9a20, 0xc010b7b680}, 0xc00d1ad700?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00a235678 sp=0xc00a235588 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc0134d2ba0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a5af770}, {0x51b9a20, 0xc010b7b680}, 0xc00d1ad7e0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00a235710 sp=0xc00a235678 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc0134d2b10}, 0xc00a5af770, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00a235790 sp=0xc00a235710 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x0?, {0x624a948?, 0xc0134d2b10?}, 0xc0081a6b00?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00a235818 sp=0xc00a235790 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc015feff20, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00a235ab8 sp=0xc00a235818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc015feff20, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00a235bf0 sp=0xc00a235ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00a236338 sp=0xc00a235bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00a236c38 sp=0xc00a236338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00a2374a0 sp=0xc00a236c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e30c3ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00a2378c0 sp=0xc00a2374a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc006134bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00a237b18 sp=0xc00a2378c0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLocked(0xc006134850, {0x624a948, 0xc0134d2ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:448 +0x388 fp=0xc00a237da8 sp=0xc00a237b18 pc=0x19e7f88 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc006134850, {0x624a948, 0xc0134d2ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:415 +0xe5 fp=0xc00a237df8 sp=0xc00a237da8 pc=0x19e7b25 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc006134850, {0x624a948, 0xc0134d2ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:377 +0xd3 fp=0xc00a237eb0 sp=0xc00a237df8 pc=0x19e7753 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc0134d2ae0?}) :1 +0x39 fp=0xc00a237ed8 sp=0xc00a237eb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc0134d2ae0}, {0x53ddbcd?, 0x1054e20?}, 0xc000055fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00a237f50 sp=0xc00a237ed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00a237fe0 sp=0xc00a237f50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a237fe8 sp=0xc00a237fe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 3727 [select]: runtime.gopark(0xc008d80d68?, 0x2?, 0x5?, 0x0?, 0xc008d80d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008d80bb8 sp=0xc008d80b98 pc=0x4907d6 runtime.selectgo(0xc008d80d68, 0xc008d80d30, 0x8?, 0x0, 0x46fe46?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008d80cf8 sp=0xc008d80bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0049d9db0, {0xc001feac58, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc008d80d98 sp=0xc008d80cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0049d9db0, {0xc001feac58?, 0xc0072f9fe0?, 0xc008d80de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc008d80dd8 sp=0xc008d80d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c79ec0, {0xc001feac58?, 0xc008d80e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc008d80e20 sp=0xc008d80dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc002c79ec0}, {0xc001feac58, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc008d80e68 sp=0xc008d80e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc004e48a20, {0xc001feac58, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc008d80ea8 sp=0xc008d80e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc001feac48, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc008d80f28 sp=0xc008d80ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc003975028?, 0xc004e48a20, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc008d80fc0 sp=0xc008d80f28 pc=0xc19546 google.golang.org/grpc.recv(0x20?, {0x7f275f301038, 0xc0000af170}, 0x484945?, {0x0?, 0x0?}, {0x51d3160, 0xc00e1c58c0}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc008d81038 sp=0xc008d80fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc001fec630, {0x51d3160?, 0xc00e1c58c0}, 0x1?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc008d81140 sp=0xc008d81038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc008d81170 sp=0xc008d81140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc004e48480, 0xc008d81248, 0xc008d81218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc008d811c8 sp=0xc008d81170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc004e48480, {0x51d3160?, 0xc00e1c58c0?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc008d81278 sp=0xc008d811c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00098b7e0, {0x51d3160?, 0xc00e1c58c0?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc008d812c0 sp=0xc008d81278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc0008aed00) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc008d812f0 sp=0xc008d812c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc008d81310 sp=0xc008d812f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0049d9bd0, 0xc008d81780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc008d813b8 sp=0xc008d81310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc001f7bb00?}, {{0xc00489d1c0, 0x2, 0x8}, {0xc00489d1c8, 0x2, 0x8}}, {0x176009f82a48f0eb, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc008d819a0 sp=0xc008d813b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc001f7bb00}, 0xc0049d99f0, 0xc00489d1c0?, {{0xc00489d1c0, 0x2, 0x8}, {0xc00489d1c8, 0x2, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc008d81e80 sp=0xc008d819a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc001f7bb00?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc008d81f58 sp=0xc008d81e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc008d81f78 sp=0xc008d81f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc008d81fe0 sp=0xc008d81f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008d81fe8 sp=0xc008d81fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 3728 [select, 9 minutes]: runtime.gopark(0xc0032b5fb0?, 0x2?, 0xc0?, 0xd1?, 0xc0032b5f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b5e18 sp=0xc0032b5df8 pc=0x4907d6 runtime.selectgo(0xc0032b5fb0, 0xc0032b5f90, 0xc0032b5fd0?, 0x0, 0x6221ba0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b5f58 sp=0xc0032b5e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0032b5fe0 sp=0xc0032b5f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b5fe8 sp=0xc0032b5fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 3729 [select, 9 minutes]: runtime.gopark(0xc0032b6fb0?, 0x2?, 0x50?, 0x6e?, 0xc0032b6f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0032b6e28 sp=0xc0032b6e08 pc=0x4907d6 runtime.selectgo(0xc0032b6fb0, 0xc0032b6f98, 0x6221ba0?, 0x0, 0x624a948?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0032b6f68 sp=0xc0032b6e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0032b6fe0 sp=0xc0032b6f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0032b6fe8 sp=0xc0032b6fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 498415 [semacquire]: runtime.gopark(0xae1b73de8ed?, 0x0?, 0x0?, 0x0?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01058b708 sp=0xc01058b6e8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x34?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc01058b770 sp=0xc01058b708 pc=0x4a17de sync.runtime_SemacquireMutex(0x0?, 0xe0?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc01058b7a0 sp=0xc01058b770 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x7f271e737420?, 0xc00cde35c0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc01058b7f8 sp=0xc01058b7a0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc0109fc000?, 0xc01058b888?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc01058b840 sp=0xc01058b7f8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0109fc000, {0x6?, 0x6?}, 0x0, 0xc008aba578, 0xc004dd0310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc01058b940 sp=0xc01058b840 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc008aba300, 0xc0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc01058b9e0 sp=0xc01058b940 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc008aba300, {0xc0014cabc0, 0x1f, 0x20}, 0xc003d3eb40?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc01058ba90 sp=0xc01058b9e0 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc003d3eb40?, 0x0?, 0x0?}, {0xc0014cabc0?, 0x1f?, 0xc00548bce0?}, 0x30?, 0x1a?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc01058baf8 sp=0xc01058ba90 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc004dd18e0?, {0xc003d3eb40?, 0x1?, 0x20?}, {0xc0014cabc0?, 0x18?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc01058bb48 sp=0xc01058baf8 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc004dd18e0, {0xc003d3eb40, 0x1f, 0x40}, {0xc0014cabc0, 0x1f, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc01058bb98 sp=0xc01058bb48 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc004dd0520, {0xc0014cabc0?, 0x0?, 0x0?}, 0x0?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc01058bcc8 sp=0xc01058bb98 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc004dd0520, {0xc003d3eb40?, 0x0?, 0x0?}, {0xc0014cabc0?, 0x0?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc01058bd08 sp=0xc01058bcc8 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc004dd0388, {0xc003d3eb40?, 0x0?, 0x20?}, {0xc0014cabc0, 0x1f, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc01058bd60 sp=0xc01058bd08 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc004dd0000, {0xc0014cabc0, 0x1f, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc01058bdf8 sp=0xc01058bd60 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc00b3a0008, {{0xc007fdbfc0, 0x1e, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc01058be48 sp=0xc01058bdf8 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc014956480, {0x624a948, 0xc040030ba0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc01058beb0 sp=0xc01058be48 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc040030ba0}, {0x62b4fe0, 0xc00b3a0008}, {0xc007fdbfc0, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc01058c480 sp=0xc01058beb0 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc040030ba0}, {0x628f760?, 0xc00b3a0000?}, {0xc007fdbfc0, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc01058c600 sp=0xc01058c480 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc040030ba0?}, {0x628f760?, 0xc00b3a0000?}, {0xc007fdbfc0?, 0xc01058c7a0?, 0x4618e7?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc01058c6f0 sp=0xc01058c600 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0?, {0x624a948, 0xc040030ba0}, {0x628f760, 0xc00b3a0000}, {0x35, 0x18, 0xc3, 0x1f, 0x7b, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:95 +0xd3 fp=0xc01058c7b0 sp=0xc01058c6f0 pc=0x1c96133 github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0x35, 0x18, 0xc3, 0x1f, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:2034 +0xbc fp=0xc01058c998 sp=0xc01058c7b0 pc=0x1d8fb1c github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc010617c20, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:206 +0x36f fp=0xc01058e808 sp=0xc01058c998 pc=0x1dc43af github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc01058f278, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc01058ee78 sp=0xc01058e808 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc006f10000, {0x624a948, 0xc040030ba0}, 0xc010617c20, 0xc00ed128c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc01058fb08 sp=0xc01058ee78 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc040030ba0}, 0xc010617c20, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc01058fe28 sp=0xc01058fb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76a5521f84, 0x0, 0x0}, 0x0, {0x17600a76e3e77f8e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc0105901f0 sp=0xc01058fe28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76a5521f84, 0x0, 0x0}, 0x0, {0x17600a76e3e77f8e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc010590b78 sp=0xc0105901f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3e77f8e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc010590dc0 sp=0xc010590b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc040030b40}, {0xc010591470?}, 0xc010617b30) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc010591430 sp=0xc010590dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc040030a80}, 0xc010617b30) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0105914d8 sp=0xc010591430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc040030a80}, {0x52f0080?, 0xc010617b30}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc010591518 sp=0xc0105914d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc040030a80}, {0x52f0080, 0xc010617b30}, 0xc003402c20, 0xc004feb7a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc010591848 sp=0xc010591518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc040030a80?}, {0x52f0080?, 0xc010617b30?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc010591888 sp=0xc010591848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc040030a80}, {0x52f0080, 0xc010617b30}, 0xc010591920?, 0xc03bfb2040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0105918b8 sp=0xc010591888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc040030a80?}, {0x52f0080?, 0xc010617b30?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0105918f8 sp=0xc0105918b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc040030a80?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc010591930 sp=0xc0105918f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc040030a80}, {0x7f278cfaa108?, 0x40?}, 0xc0105919e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0105919a8 sp=0xc010591930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc040030a80?}, {0x52f0080?, 0xc010617b30?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc010591a20 sp=0xc0105919a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc040030a80?}, {0x52f0080?, 0xc010617b30?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc010591a60 sp=0xc010591a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc040030a80}, {0x52f0080, 0xc010617b30}, 0xc003402c20, 0xc004feb7a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc010591ab0 sp=0xc010591a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc040030a80}, 0xc011f07140, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc010591b08 sp=0xc010591ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc003f74120, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc010591e48 sp=0xc010591b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc003f74120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc010591f68 sp=0xc010591e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc010591fe0 sp=0xc010591f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010591fe8 sp=0xc010591fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498366 [select]: runtime.gopark(0xc00a62fc18?, 0x3?, 0xd2?, 0xf9?, 0xc00a62fb3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a62f8f0 sp=0xc00a62f8d0 pc=0x4907d6 runtime.selectgo(0xc00a62fc18, 0xc00a62fb34, 0xc00a1d6780?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a62fa30 sp=0xc00a62f8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc012b41050}, {0x6221560, 0xc032e03d40}, {0x46?, {0xc00a62fe01?, 0xa62fe60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00a62fda0 sp=0xc00a62fa30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc012b41050?}, 0x164, 0xc032e03d40, 0x1f?, 0xc00d57eac0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00a62fe70 sp=0xc00a62fda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc012b41050}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00a62ff30 sp=0xc00a62fe70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a62ffe0 sp=0xc00a62ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a62ffe8 sp=0xc00a62ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3800 [sync.Cond.Wait]: runtime.gopark(0xc00d1656d0?, 0xc01e6e1c98?, 0x0?, 0x55?, 0xc01745b578?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ce45c78 sp=0xc00ce45c58 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc000b70b78, 0x1dfe) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00ce45cc0 sp=0xc00ce45c78 pc=0x4be30c sync.(*Cond).Wait(0x5?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00ce45cf8 sp=0xc00ce45cc0 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000b70b60, {0x624a8a0, 0xc0037454c0}, 0xa18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb fp=0xc00ce45db0 sp=0xc00ce45cf8 pc=0x1d1cc5b github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x624a948?, 0xc0037fa720?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 fp=0xc00ce45f30 sp=0xc00ce45db0 pc=0x1d1dbd6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ce45fe0 sp=0xc00ce45f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ce45fe8 sp=0xc00ce45fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 120615 [select]: runtime.gopark(0xc00aa27950?, 0x4?, 0x46?, 0xfe?, 0xc00aa278a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00aa27700 sp=0xc00aa276e0 pc=0x4907d6 runtime.selectgo(0xc00aa27950, 0xc00aa27898, 0xc0111d0d00?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00aa27840 sp=0xc00aa27700 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000c62700, 0xc0161b1c60, {0x628a900, 0xc0161b1d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:434 +0x345 fp=0xc00aa27cd8 sp=0xc00aa27840 pc=0x1d83c25 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x624a948, 0xc0111d7590}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:601 +0x367 fp=0xc00aa27e18 sp=0xc00aa27cd8 pc=0x1d852e7 runtime/pprof.Do({0x624a948?, 0xc0111d7530?}, {{0xc016717a20?, 0xc0046a1eb8?, 0xc448e6?}}, 0xc016717a00) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00aa27e88 sp=0xc00aa27e18 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x624a948, 0xc0111d7530}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:607 +0x1c8 fp=0xc00aa27f30 sp=0xc00aa27e88 pc=0x1d84f28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00aa27fe0 sp=0xc00aa27f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00aa27fe8 sp=0xc00aa27fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 4120 [IO wait]: runtime.gopark(0x0?, 0xb?, 0x0?, 0x0?, 0xa5b?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0034dbaa0 sp=0xc0034dba80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0xbef6f1?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0034dbad8 sp=0xc0034dbaa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbccb8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0034dbaf8 sp=0xc0034dbad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc004f1a480?, 0xc0058a4000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0034dbb20 sp=0xc0034dbaf8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc004f1a480, {0xc0058a4000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0034dbba0 sp=0xc0034dbb20 pc=0x535efa net.(*netFD).Read(0xc004f1a480, {0xc0058a4000?, 0xc0034dbc70?, 0x46159f?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0034dbbe8 sp=0xc0034dbba0 pc=0x7a20c9 net.(*conn).Read(0xc005168608, {0xc0058a4000?, 0x7f278cfaa108?, 0x18?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0034dbc30 sp=0xc0034dbbe8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc004d8df80, {0xc0058a4000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc0034dbc78 sp=0xc0034dbc30 pc=0x1641f96 bufio.(*Reader).Read(0xc0066e80c0, {0xc004b6a120, 0x9, 0xc054b7d548?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc0034dbcb0 sp=0xc0034dbc78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc0066e80c0}, {0xc004b6a120, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc0034dbcf8 sp=0xc0034dbcb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004b6a120?, 0x9?, 0xc000198110?}, {0x62052c0?, 0xc0066e80c0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc0034dbd48 sp=0xc0034dbcf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc004b6a0e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc0034dbdf8 sp=0xc0034dbd48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc007f59ba0, 0x0?, 0x1054c46?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc0034dbf18 sp=0xc0034dbdf8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc007f59ba0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc0034dbf80 sp=0xc0034dbf18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc0034dbfe0 sp=0xc0034dbf80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0034dbfe8 sp=0xc0034dbfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 344461 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x2e3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a9f0f58 sp=0xc00a9f0f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x645399f0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00a9f0f90 sp=0xc00a9f0f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539908, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00a9f0fb0 sp=0xc00a9f0f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc005903d80?, 0xc007235000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00a9f0fd8 sp=0xc00a9f0fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc005903d80, {0xc007235000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00a9f1058 sp=0xc00a9f0fd8 pc=0x535efa net.(*netFD).Read(0xc005903d80, {0xc007235000?, 0x7b5a68?, 0xc005903d80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00a9f10a0 sp=0xc00a9f1058 pc=0x7a20c9 net.(*conn).Read(0xc007f0a060, {0xc007235000?, 0x164290f?, 0xc007f0a060?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00a9f10e8 sp=0xc00a9f10a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002e957a0, {0xc007235000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00a9f1130 sp=0xc00a9f10e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc001c20870, {0xc007235000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00a9f11d0 sp=0xc00a9f1130 pc=0x370f896 bufio.(*Reader).fill(0xc003864890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00a9f1208 sp=0xc00a9f11d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003864890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00a9f1228 sp=0xc00a9f1208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003864e68?, {0x62295c8, 0xc003864890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00a9f1260 sp=0xc00a9f1228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003864800, 0xc010cbe890, 0xc00a9f1442, 0xc00a9f1444, {0x623b718, 0xc011773d40}, 0xc0059fa000, 0xc006a50254, 0xc00a9f1440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00a9f1398 sp=0xc00a9f1260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003864800, {0x624a948, 0xc008b29cb0}, 0xc010cbe880, 0xc0059fa000, 0xc008b29b30, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00a9f1768 sp=0xc00a9f1398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc008b29cb0}, {0x628a538, 0xc002e957a0}, {{{0xc00d58a038, 0x4}}, 0x1, 0xc008b29b60, 0xc008b29bf0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00a9f18a8 sp=0xc00a9f1768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc008b29cb0}, {0x628a538?, 0xc002e957a0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00a9f1e20 sp=0xc00a9f18a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc008b299b0?}, {0x628a538, 0xc002e957a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00a9f1eb0 sp=0xc00a9f1e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc008b299b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00a9f1f30 sp=0xc00a9f1eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a9f1fe0 sp=0xc00a9f1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a9f1fe8 sp=0xc00a9f1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498466 [select]: runtime.gopark(0xc00a3d9c20?, 0x3?, 0x22?, 0x9b?, 0xc00a3d9bda?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a3d9a40 sp=0xc00a3d9a20 pc=0x4907d6 runtime.selectgo(0xc00a3d9c20, 0xc00a3d9bd4, 0xc0088aa1e0?, 0x0, 0xc010c4b400?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a3d9b80 sp=0xc00a3d9a40 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc005b2af20, {0x624a910, 0xc0088aa1e0}, 0xc00a3d9ca8?, {0x629caa8, 0xc011e78700}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x165 fp=0xc00a3d9c60 sp=0xc00a3d9b80 pc=0x1ca0605 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0005593b0, {0x624a910, 0xc0088aa1e0}, 0xc00a3d9dd0?, 0xc00dd62d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:749 +0x3d1 fp=0xc00a3d9da8 sp=0xc00a3d9c60 pc=0x1ca75d1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x624a910?, 0xc0088aa1e0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:792 +0x38 fp=0xc00a3d9de0 sp=0xc00a3d9da8 pc=0x1ca7bd8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x624a948?, 0xc004a201e0?}, {0x537e81f, 0x12}, 0x4a817c800, 0xc00a3d9f00) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed fp=0xc00a3d9e80 sp=0xc00a3d9de0 pc=0x16210cd github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x624a948, 0xc004a201e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:790 +0xac fp=0xc00a3d9f30 sp=0xc00a3d9e80 pc=0x1ca7aac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a3d9fe0 sp=0xc00a3d9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a3d9fe8 sp=0xc00a3d9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498493 [select]: runtime.gopark(0xc0110d9208?, 0x2?, 0x40?, 0xb1?, 0xc0110d9204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0110d9090 sp=0xc0110d9070 pc=0x4907d6 runtime.selectgo(0xc0110d9208, 0xc0110d9200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0110d91d0 sp=0xc0110d9090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc058a58fc0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0110d9238 sp=0xc0110d91d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030827340, {0x51b9a20?, 0xc0533f6c00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0110d9340 sp=0xc0110d9238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc8?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0110d9370 sp=0xc0110d9340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc058a58b40, 0xc0110d9448, 0xc0110d9418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0110d93c8 sp=0xc0110d9370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc058a58b40, {0x51b9a20?, 0xc0533f6c00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0110d9478 sp=0xc0110d93c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00aeb3980?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc017227e00}, {0x51b9a20, 0xc0533f6c00}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0110d94e0 sp=0xc0110d9478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb3980}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227e00}, {0x51b9a20, 0xc0533f6c00}, 0x1322657102bbcaf8?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0110d95d0 sp=0xc0110d94e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00aeb3980?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc017227e00?}, {0x51b9a20?, 0xc0533f6c00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc0110d9670 sp=0xc0110d95d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277510, {0x624a948, 0xc00aeb3980}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0110d96f0 sp=0xc0110d9670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277510?}}, {0x624a948, 0xc00aeb3980}, 0xc017227e00, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0110d97c8 sp=0xc0110d96f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00aeb3980?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc0110d9818 sp=0xc0110d97c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00ccfd920, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0110d9ab8 sp=0xc0110d9818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00ccfd920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51258bb, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0110d9bf0 sp=0xc0110d9ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51258bb, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0110da338 sp=0xc0110d9bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51258bb, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0110dac38 sp=0xc0110da338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51258bb, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0110db4a0 sp=0xc0110dac38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51258bb, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0110db8c0 sp=0xc0110db4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0110db9f8 sp=0xc0110db8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00aeb2240}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc0110dbc00 sp=0xc0110db9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00aeb2240}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc0110dbf30 sp=0xc0110dbc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0110dbfe0 sp=0xc0110dbf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0110dbfe8 sp=0xc0110dbfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 429791 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1c2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b46ef58 sp=0xc00b46ef38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7900?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00b46ef90 sp=0xc00b46ef58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7818, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00b46efb0 sp=0xc00b46ef90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc004fb9700?, 0xc00851e000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00b46efd8 sp=0xc00b46efb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc004fb9700, {0xc00851e000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00b46f058 sp=0xc00b46efd8 pc=0x535efa net.(*netFD).Read(0xc004fb9700, {0xc00851e000?, 0x7b5a68?, 0xc004fb9700?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00b46f0a0 sp=0xc00b46f058 pc=0x7a20c9 net.(*conn).Read(0xc016884568, {0xc00851e000?, 0x164290f?, 0xc016884568?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00b46f0e8 sp=0xc00b46f0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03d859920, {0xc00851e000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00b46f130 sp=0xc00b46f0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc013d82fa8, {0xc00851e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00b46f1d0 sp=0xc00b46f130 pc=0x370f896 bufio.(*Reader).fill(0xc00a6dd090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00b46f208 sp=0xc00b46f1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00a6dd090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00b46f228 sp=0xc00b46f208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00a6dd668?, {0x62295c8, 0xc00a6dd090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00b46f260 sp=0xc00b46f228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00a6dd000, 0xc0083ef0e0, 0xc00b46f442, 0xc00b46f444, {0x623b718, 0xc043289170}, 0xc0059fa000, 0xc00754dbd4, 0xc00b46f440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00b46f398 sp=0xc00b46f260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00a6dd000, {0x624a948, 0xc005c26270}, 0xc0083ef0d0, 0xc0059fa000, 0xc005c260f0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00b46f768 sp=0xc00b46f398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc005c26270}, {0x628a538, 0xc03d859920}, {{{0xc00c219038, 0x4}}, 0x1, 0xc005c26120, 0xc005c26210, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00b46f8a8 sp=0xc00b46f768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc005c26270}, {0x628a538?, 0xc03d859920}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00b46fe20 sp=0xc00b46f8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc005c26000?}, {0x628a538, 0xc03d859920}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00b46feb0 sp=0xc00b46fe20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc005c26000}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00b46ff30 sp=0xc00b46feb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b46ffe0 sp=0xc00b46ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b46ffe8 sp=0xc00b46ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 404930 [runnable]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x3f8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f338f58 sp=0xc00f338f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbd9d0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00f338f90 sp=0xc00f338f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd8e8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00f338fb0 sp=0xc00f338f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0468f6000?, 0xc00fa70000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00f338fd8 sp=0xc00f338fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0468f6000, {0xc00fa70000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00f339058 sp=0xc00f338fd8 pc=0x535efa net.(*netFD).Read(0xc0468f6000, {0xc00fa70000?, 0x7b5a68?, 0xc0468f6000?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00f3390a0 sp=0xc00f339058 pc=0x7a20c9 net.(*conn).Read(0xc012084f30, {0xc00fa70000?, 0x164290f?, 0xc012084f30?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00f3390e8 sp=0xc00f3390a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00dfc0840, {0xc00fa70000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00f339130 sp=0xc00f3390e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01285ec78, {0xc00fa70000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00f3391d0 sp=0xc00f339130 pc=0x370f896 bufio.(*Reader).fill(0xc003491c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00f339208 sp=0xc00f3391d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003491c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00f339228 sp=0xc00f339208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003492268?, {0x62295c8, 0xc003491c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00f339260 sp=0xc00f339228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003491c00, 0xc0172dc710, 0xc00f339442, 0xc00f339444, {0x623b718, 0xc007572630}, 0xc0059fa000, 0xc010f07cc4, 0xc00f339440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00f339398 sp=0xc00f339260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003491c00, {0x624a948, 0xc00703a450}, 0xc0172dc6f0, 0xc0059fa000, 0xc00703a300, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00f339768 sp=0xc00f339398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc00703a450}, {0x628a538, 0xc00dfc0840}, {{{0xc010cce022, 0x4}}, 0x1, 0xc00703a330, 0xc00703a360, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00f3398a8 sp=0xc00f339768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc00703a450}, {0x628a538?, 0xc00dfc0840}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00f339e20 sp=0xc00f3398a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00703a240?}, {0x628a538, 0xc00dfc0840}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00f339eb0 sp=0xc00f339e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00703a240}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00f339f30 sp=0xc00f339eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f339fe0 sp=0xc00f339f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f339fe8 sp=0xc00f339fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 407460 [runnable]: github.com/cockroachdb/cockroach/pkg/util.(*FastIntSet).Add(0xc0230ef738?, 0x9?) github.com/cockroachdb/cockroach/pkg/util/pkg/util/fast_int_set.go:94 +0xd4 fp=0xc0230ef650 sp=0xc0230ef648 pc=0x944634 github.com/cockroachdb/cockroach/pkg/sql/opt.(*ColSet).Add(...) github.com/cockroachdb/cockroach/pkg/sql/opt/colset.go:60 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*CustomFuncs).SimplifiablePartialIndexProjectCols(0xc003900240, 0xc01705e638, {0x9c5fde0, 0x0, 0x0}, {0x9c5fde0, 0x0, 0x0}, {0xc011ea0d80, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/sql/opt/norm/mutation_funcs.go:36 +0x179 fp=0xc0230ef7c0 sp=0xc0230ef650 pc=0x24a2859 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).ConstructUpdate(0xc003900230, {0x62a4240?, 0xc019ddcfb8}, {0x9c5fde0, 0x0, 0x0}, {0x9c5fde0, 0x0, 0x0}, 0xc01705e638) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/norm/factory.og.go:190 +0x13f fp=0xc0230ef8a8 sp=0xc0230ef7c0 pc=0x24b833f github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).CopyAndReplaceDefault(0xc003900230, {0x626f840?, 0xc01705e5f8?}, 0x0?) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/norm/factory.og.go:23723 +0x4e53 fp=0xc0230efaa8 sp=0xc0230ef8a8 pc=0x253d653 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).AssignPlaceholders.func2({0x626f840?, 0xc01705e5f8?}) github.com/cockroachdb/cockroach/pkg/sql/opt/norm/factory.go:376 +0x15b fp=0xc0230efba8 sp=0xc0230efaa8 pc=0x24918db github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).invokeReplace(0xc0230efc38?, {0x626f840, 0xc01705e5f8}, 0xc0230efff8) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/norm/factory.og.go:25099 +0x77 fp=0xc0230efbe8 sp=0xc0230efba8 pc=0x2547a37 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).CopyAndReplaceDefault(0xc003900230, {0x626ea80?, 0xc0060277f8?}, 0x0?) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/norm/factory.og.go:23786 +0x8db7 fp=0xc0230efde8 sp=0xc0230efbe8 pc=0x25415b7 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).AssignPlaceholders.func2({0x626ea80?, 0xc0060277f8?}) github.com/cockroachdb/cockroach/pkg/sql/opt/norm/factory.go:376 +0x15b fp=0xc0230efee8 sp=0xc0230efde8 pc=0x24918db github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).invokeReplace(0xc003d69400?, {0x626ea80, 0xc0060277f8}, 0xc0230efff8) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/norm/factory.og.go:25099 +0x77 fp=0xc0230eff28 sp=0xc0230efee8 pc=0x2547a37 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).CopyAndReplace(0xc003900230, {0x62a4240, 0xc0060277f8}, 0x49e8ec0?, 0x1?) github.com/cockroachdb/cockroach/pkg/sql/opt/norm/factory.go:313 +0x110 fp=0xc0230eff98 sp=0xc0230eff28 pc=0x24914d0 github.com/cockroachdb/cockroach/pkg/sql/opt/norm.(*Factory).AssignPlaceholders(0xc003900230, 0xc00c2fce00) github.com/cockroachdb/cockroach/pkg/sql/opt/norm/factory.go:378 +0x105 fp=0xc0230f0028 sp=0xc0230eff98 pc=0x2491705 github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).reuseMemo(0xc0039001a8, {0x624a948?, 0xc040031f20?}, 0xc00c2fce00) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:489 +0x76 fp=0xc0230f0060 sp=0xc0230f0028 pc=0x347cd96 github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).buildExecMemo(0xc0039001a8, {0x624a948?, 0xc040031f20}) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:522 +0x171 fp=0xc0230f0160 sp=0xc0230f0060 pc=0x347cfb1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).makeOptimizerPlan(0xc0038ff878, {0x624a8a0?, 0xc03bfb27c0?}) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:232 +0x1b1 fp=0xc0230f0258 sp=0xc0230f0160 pc=0x347bef1 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).makeExecPlan(0xc0038ff300, {0x624a8a0, 0xc03bfb27c0}, 0xc0038ff878) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1473 +0x4e fp=0xc0230f0300 sp=0xc0230f0258 pc=0x331046e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0038ff300, {0x624a8a0, 0xc03bfb27c0}, 0xc0038ff878, {0x7f275f329680, 0xc00d3b3ac8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1084 +0x314 fp=0xc0230f0720 sp=0xc0230f0300 pc=0x330d774 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0038ff300, {0x624a948?, 0xc03d7fd020?}, {{0x6274800, 0xc011248280}, {0xc010a9331b, 0x8d}, 0x3, 0x1}, 0xc011494ea0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0230f10b0 sp=0xc0230f0720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc03d7fd020?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc0230f1168 sp=0xc0230f10b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc03d7fd020?}, {0x6274800?, 0xc011248280?}, 0x624a948?, 0xc03d7fd020?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0230f12b8 sp=0xc0230f1168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0038ff300, {0x624a948, 0xc03d7fd020}, {{0x6274800, 0xc011248280}, {0xc010a9331b, 0x8d}, 0x3, 0x1}, 0xc011494ea0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc0230f1430 sp=0xc0230f12b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0038ff300, {0x624a948, 0xc03d7fd020}, {0xc011494ea0, {0xc040031830, 0x3, 0x3}, {0xc0040c7ba0, 0x6, 0x6}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc0230f1550 sp=0xc0230f1430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc003070f57, 0x0}, 0x0, {0xc1116e81980dbedb, 0x7fedfced29, 0x0}, 0x1}, 0xc0038ff300, 0xc0230f18c0, 0xc0230f18b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc0230f1708 sp=0xc0230f1550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0038ff300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc0230f1cb8 sp=0xc0230f1708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0038ff300, {0x624a8a0, 0xc00b8bd780}, 0x411d675000000000?, 0xc003f53dc0?, 0xc00be50520?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0230f1d70 sp=0xc0230f1cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00b8bd780?}, {0xc0059fa000?}, 0x3?, 0xc004c9e138?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0230f1e20 sp=0xc0230f1d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0230f1fe0 sp=0xc0230f1e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0230f1fe8 sp=0xc0230f1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 440698 [select]: runtime.gopark(0xc016350768?, 0x2?, 0x70?, 0xf1?, 0xc016350764?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0163505f0 sp=0xc0163505d0 pc=0x4907d6 runtime.selectgo(0xc016350768, 0xc016350760, 0x10?, 0x0, 0x301?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc016350730 sp=0xc0163505f0 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0037099e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc016350798 sp=0xc016350730 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ec98bb0, {0x51b9a20?, 0xc010b7bd00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0163508a0 sp=0xc016350798 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x34c?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0163508d0 sp=0xc0163508a0 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0037097a0, 0xc0163509a8, 0xc016350978) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc016350928 sp=0xc0163508d0 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0037097a0, {0x51b9a20?, 0xc010b7bd00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0163509d8 sp=0xc016350928 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc016171da0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003dfe5a0}, {0x51b9a20, 0xc010b7bd00}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc016350a40 sp=0xc0163509d8 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc016171da0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dfe5a0}, {0x51b9a20, 0xc010b7bd00}, 0x21ca8d8646b7a89?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc016350b30 sp=0xc016350a40 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x2b53ed818fe90477?, {0x624a948?, 0xc016171da0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003dfe5a0?}, {0x51b9a20?, 0xc010b7bd00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc016350bd0 sp=0xc016350b30 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123bef98, {0x624a948, 0xc016171da0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc016350c50 sp=0xc016350bd0 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123bef98?}}, {0x624a948, 0xc016171da0}, 0xc003dfe5a0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc016350d28 sp=0xc016350c50 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x16350d90?, {0x624a948?, 0xc016171da0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc016350e88?}) :1 +0x65 fp=0xc016350d78 sp=0xc016350d28 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0113e8360, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc016351018 sp=0xc016350d78 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0113e8360, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3628705, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc016351150 sp=0xc016351018 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3628705, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc016351898 sp=0xc016351150 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3628705, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc016352198 sp=0xc016351898 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3628705, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc016352a00 sp=0xc016352198 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3628705, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc016352e20 sp=0xc016352a00 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc007500bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc016353078 sp=0xc016352e20 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc007500b80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc0163531a0 sp=0xc016353078 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc007500b50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc016353430 sp=0xc0163531a0 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc007500a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc0163539d0 sp=0xc016353430 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc007500a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc016353cc0 sp=0xc0163539d0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc007500920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc016353fc0 sp=0xc016353cc0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc007500900?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc016354168 sp=0xc016353fc0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc007500850, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc016354380 sp=0xc016354168 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc007500680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc0163545d8 sp=0xc016354380 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc016354700 sp=0xc0163545d8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc005a134a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc0163549c8 sp=0xc016354700 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc016354ad8 sp=0xc0163549c8 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc00e10b6b0}, 0x8?, 0xc0225a2100) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc016354cd0 sp=0xc016354ad8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc005a134a0, {0x624a948, 0xc00e10b6b0}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc016354d18 sp=0xc016354cd0 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc004d4ca28, {0x624a948, 0xc00e10b6b0}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc016354d50 sp=0xc016354d18 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc004d4ca00, {{0x624a948, 0xc00e10b6b0}, 0xc010cc4000, 0xc0037eab78}) github.com/cockroachdb/cockroach/pkg/sql/update.go:197 +0x225 fp=0xc016354db0 sp=0xc016354d50 pc=0x352d9a5 github.com/cockroachdb/cockroach/pkg/sql.(*serializeNode).Next(0xc003d0b950, {{0x624a948?, 0xc00e10b6b0?}, 0xc010cc4000?, 0xc0037eab78?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:115 +0x5a fp=0xc016354de8 sp=0xc016354db0 pc=0x3476ffa github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc001a1bb80) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc016354ee0 sp=0xc016354de8 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc007deb800) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:243 +0x13c fp=0xc016355260 sp=0xc016354ee0 pc=0x299d6fc github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc00a5c84d0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:202 +0x2e fp=0xc0163552b0 sp=0xc016355260 pc=0x2860bee github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:48 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc010eaea90) github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:195 +0x4a fp=0xc016355410 sp=0xc0163552b0 pc=0x299be6a github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0142d36e0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f fp=0xc0163554d0 sp=0xc016355410 pc=0x293bf5f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc008bd6160) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc0163554e8 sp=0xc0163554d0 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc016355500 sp=0xc0163554e8 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc003bc2e00?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc016355540 sp=0xc016355500 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc008bd6160, {0x624a8a0?, 0xc00c8f7b00?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc0163556d0 sp=0xc016355540 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0100c3d10, {0x624a8a0, 0xc00c8f7b00}, 0xc005a134a0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc016355770 sp=0xc0163556d0 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc003d0bce0}, 0xc00dee5b30, 0xc005a134a0, 0xc01302b680, 0xc003999500, 0xc0037eaeb0, 0xc0005c7110) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc016356020 sp=0xc016355770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc003d0bce0}, 0xe8?, 0xc00dee5b30, 0x8deee115f6b9ca01?, {{0x624ce40?, 0xc0305181e0?}, 0x0?}, 0xc003999500, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0163560e0 sp=0xc016356020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc003d0bce0}, 0xc0037eaeb0, 0xc0037eab78?, 0xc0037eab78, 0xc003999500, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc0163561f8 sp=0xc0163560e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0037ea600, {0x624a948?, 0xc003d0bce0}, 0xc0037eab78, 0xc003d0bce0?, {0x7f275f329680?, 0xc00403fec8}, 0x270?, 0xc01d3a4d08) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc016356300 sp=0xc0163561f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0037ea600, {0x624a8a0, 0xc00c8f6880}, 0xc0037eab78, {0x7f275f329680, 0xc00403fec8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc016356720 sp=0xc016356300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0037ea600, {0x624a948?, 0xc0470da5a0?}, {{0x6274800, 0xc012083900}, {0xc00153400d, 0x270}, 0x6, 0x1}, 0xc007eafc20, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0163570b0 sp=0xc016356720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc0470da5a0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc016357168 sp=0xc0163570b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc0470da5a0?}, {0x6274800?, 0xc012083900?}, 0x624a948?, 0xc0470da5a0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0163572b8 sp=0xc016357168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0037ea600, {0x624a948, 0xc0470da5a0}, {{0x6274800, 0xc012083900}, {0xc00153400d, 0x270}, 0x6, 0x1}, 0xc007eafc20, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc016357430 sp=0xc0163572b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0037ea600, {0x624a948, 0xc0470da5a0}, {0xc007eafc20, {0xc0142d32c0, 0x6, 0x6}, {0xc006024b20, 0xf, 0xf}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc016357550 sp=0xc016357430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0027ac9a2, 0x0}, 0x0, {0xc1116e818d69693a, 0x7fe358978f, 0x0}, 0x1}, 0xc0037ea600, 0xc0163578c0, 0xc0163578b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc016357708 sp=0xc016357550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0037ea600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc016357cb8 sp=0xc016357708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0037ea600, {0x624a8a0, 0xc005b1c380}, 0x411a1b8000000000?, 0xc003f4fdc0?, 0xc00faa2ad0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc016357d70 sp=0xc016357cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc005b1c380?}, {0xc0059fa000?}, 0x3?, 0xc004985320?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc016357e20 sp=0xc016357d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc016357fe0 sp=0xc016357e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016357fe8 sp=0xc016357fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498496 [select]: runtime.gopark(0xc011a21208?, 0x2?, 0x98?, 0xb3?, 0xc011a21204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011a21090 sp=0xc011a21070 pc=0x4907d6 runtime.selectgo(0xc011a21208, 0xc011a21200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc011a211d0 sp=0xc011a21090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0052ee000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc011a21238 sp=0xc011a211d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0308276b0, {0x51b9a20?, 0xc0533f6d00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc011a21340 sp=0xc011a21238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc011a21370 sp=0xc011a21340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc058a598c0, 0xc011a21448, 0xc011a21418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc011a213c8 sp=0xc011a21370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc058a598c0, {0x51b9a20?, 0xc0533f6d00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc011a21478 sp=0xc011a213c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03234b170?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003bee1e0}, {0x51b9a20, 0xc0533f6d00}, 0x0?, {0xc00a393c00, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc011a214e0 sp=0xc011a21478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03234b170}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003bee1e0}, {0x51b9a20, 0xc0533f6d00}, 0x2f668dd86f66382f?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc011a215d0 sp=0xc011a214e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc03234b170?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003bee1e0?}, {0x51b9a20?, 0xc0533f6d00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc011a21670 sp=0xc011a215d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0432775e0, {0x624a948, 0xc03234b170}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc011a216f0 sp=0xc011a21670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0432775e0?}}, {0x624a948, 0xc03234b170}, 0xc003bee1e0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc011a217c8 sp=0xc011a216f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc03234b170?}, 0x624a948?, {0x0?, 0xc005751d20?, 0xc00c88d928?}) :1 +0x65 fp=0xc011a21818 sp=0xc011a217c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc007b25ce0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc011a21ab8 sp=0xc011a21818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc007b25ce0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e515a59a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc011a21bf0 sp=0xc011a21ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e515a59a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc011a22338 sp=0xc011a21bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e515a59a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc011a22c38 sp=0xc011a22338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e515a59a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc011a234a0 sp=0xc011a22c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e515a59a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc011a238c0 sp=0xc011a234a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc011a239f8 sp=0xc011a238c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00aeb2570}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc011a23c00 sp=0xc011a239f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00aeb2570}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc011a23f30 sp=0xc011a23c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011a23fe0 sp=0xc011a23f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011a23fe8 sp=0xc011a23fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 409218 [sleep]: runtime.gopark(0x4b8ceddba7f?, 0x604ca?, 0xe2?, 0x43?, 0x80c3a3d046?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003dd3df0 sp=0xc003dd3dd0 pc=0x4907d6 time.Sleep(0x150d8824d) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003dd3e30 sp=0xc003dd3df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0037fb470, 0x604ca, {0x2?, 0x203000?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003dd3e88 sp=0xc003dd3e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0037fb470, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003dd3ec8 sp=0xc003dd3e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc005418f00?, 0x1d, 0xc0028cf770?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003dd3fa8 sp=0xc003dd3ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003dd3fe0 sp=0xc003dd3fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003dd3fe8 sp=0xc003dd3fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 486210 [sleep]: runtime.gopark(0x4b847c85401?, 0x801999?, 0xa9?, 0xa0?, 0x7ffa35cefd?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aa7df0 sp=0xc004aa7dd0 pc=0x4907d6 time.Sleep(0x10e7fe67b) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004aa7e30 sp=0xc004aa7df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc006dfaae0, 0x801999, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004aa7e88 sp=0xc004aa7e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc006dfaae0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004aa7ec8 sp=0xc004aa7e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00603f180?, 0x6, 0xc004ab3f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004aa7fa8 sp=0xc004aa7ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004aa7fe0 sp=0xc004aa7fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aa7fe8 sp=0xc004aa7fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498410 [select]: runtime.gopark(0xc00b605808?, 0x3?, 0x4?, 0x0?, 0xc00b6054d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b605258 sp=0xc00b605238 pc=0x4907d6 runtime.selectgo(0xc00b605808, 0xc00b6054cc, 0x17600a76b8973ca6?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b605398 sp=0xc00b605258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc035bf97d0}, 0xc00db29c20, 0xc047c0dc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00b605b08 sp=0xc00b605398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc035bf97d0}, 0xc00db29c20, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00b605e28 sp=0xc00b605b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76b7272fa9, 0x0, 0x0}, 0x0, {0x17600a76e16e3d12, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00b6061f0 sp=0xc00b605e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76b7272fa9, 0x0, 0x0}, 0x0, {0x17600a76e16e3d12, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00b606b78 sp=0xc00b6061f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e16e3d12, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00b606dc0 sp=0xc00b606b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc03f1f78f0}, {0xc00b607470?}, 0xc00a60f590) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00b607430 sp=0xc00b606dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc03f1f7800}, 0xc00a60f590) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00b6074d8 sp=0xc00b607430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc03f1f7800}, {0x52f0080?, 0xc00a60f590}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00b607518 sp=0xc00b6074d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc03f1f7800}, {0x52f0080, 0xc00a60f590}, 0xc03080a960, 0xc005914c78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00b607848 sp=0xc00b607518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03f1f7800?}, {0x52f0080?, 0xc00a60f590?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00b607888 sp=0xc00b607848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc03f1f7800}, {0x52f0080, 0xc00a60f590}, 0xc00b607920?, 0xc00006ed80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00b6078b8 sp=0xc00b607888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03f1f7800?}, {0x52f0080?, 0xc00a60f590?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b6078f8 sp=0xc00b6078b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc03f1f7800?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00b607930 sp=0xc00b6078f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc03f1f7800}, {0x20300c?, 0x40?}, 0xc00b6079e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00b6079a8 sp=0xc00b607930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc03f1f7800?}, {0x52f0080?, 0xc00a60f590?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00b607a20 sp=0xc00b6079a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03f1f7800?}, {0x52f0080?, 0xc00a60f590?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b607a60 sp=0xc00b607a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc03f1f7800}, {0x52f0080, 0xc00a60f590}, 0xc03080a960, 0xc005914c78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00b607ab0 sp=0xc00b607a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc03f1f7800}, 0xc00fe5cb40, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00b607b08 sp=0xc00b607ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc00cd1c360, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00b607e48 sp=0xc00b607b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc00cd1c360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00b607f68 sp=0xc00b607e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00b607fe0 sp=0xc00b607f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b607fe8 sp=0xc00b607fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 488391 [semacquire]: runtime.gopark(0x17600a74aea4a89a?, 0x0?, 0x80?, 0x36?, 0x1?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dff2f00 sp=0xc00dff2ee0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0100caf78, 0x20?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00dff2f68 sp=0xc00dff2f00 pc=0x4a17de sync.runtime_Semacquire(0xc002a40360?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00dff2f98 sp=0xc00dff2f68 pc=0x4be005 sync.(*WaitGroup).Wait(0xc00dff2f60?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00dff2fc0 sp=0xc00dff2f98 pc=0x4ce332 github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Wait.func1() github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:499 +0x2e fp=0xc00dff2fe0 sp=0xc00dff2fc0 pc=0x25bf40e runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dff2fe8 sp=0xc00dff2fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Wait github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:498 +0xd8 goroutine 492841 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x539?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cc72f58 sp=0xc00cc72f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7270?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00cc72f90 sp=0xc00cc72f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7188, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00cc72fb0 sp=0xc00cc72f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00f71f800?, 0xc003ed4000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00cc72fd8 sp=0xc00cc72fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00f71f800, {0xc003ed4000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00cc73058 sp=0xc00cc72fd8 pc=0x535efa net.(*netFD).Read(0xc00f71f800, {0xc003ed4000?, 0x7b5a68?, 0xc00f71f800?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00cc730a0 sp=0xc00cc73058 pc=0x7a20c9 net.(*conn).Read(0xc008252c90, {0xc003ed4000?, 0x164290f?, 0xc008252c90?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00cc730e8 sp=0xc00cc730a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc011c25680, {0xc003ed4000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00cc73130 sp=0xc00cc730e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc006de7dd0, {0xc003ed4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00cc731d0 sp=0xc00cc73130 pc=0x370f896 bufio.(*Reader).fill(0xc00e8d7890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00cc73208 sp=0xc00cc731d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00e8d7890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00cc73228 sp=0xc00cc73208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00e8d7e68?, {0x62295c8, 0xc00e8d7890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00cc73260 sp=0xc00cc73228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00e8d7800, 0xc00c8ded00, 0xc00cc73442, 0xc00cc73444, {0x623b718, 0xc0061d6120}, 0xc0059fa000, 0xc042bfea74, 0xc00cc73440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00cc73398 sp=0xc00cc73260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00e8d7800, {0x624a948, 0xc02a20f4d0}, 0xc00c8decf0, 0xc0059fa000, 0xc02a20f260, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00cc73768 sp=0xc00cc73398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc02a20f4d0}, {0x628a538, 0xc011c25680}, {{{0xc003e92046, 0x4}}, 0x1, 0xc02a20f2c0, 0xc02a20f350, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00cc738a8 sp=0xc00cc73768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc02a20f4d0}, {0x628a538?, 0xc011c25680}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00cc73e20 sp=0xc00cc738a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0083fab10?}, {0x628a538, 0xc011c25680}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00cc73eb0 sp=0xc00cc73e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0083fab10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00cc73f30 sp=0xc00cc73eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cc73fe0 sp=0xc00cc73f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cc73fe8 sp=0xc00cc73fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498401 [select]: runtime.gopark(0xc0124482d8?, 0x2?, 0x8?, 0xd9?, 0xc0124482d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012448160 sp=0xc012448140 pc=0x4907d6 runtime.selectgo(0xc0124482d8, 0xc0124482d0, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0124482a0 sp=0xc012448160 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00bc43560) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc012448308 sp=0xc0124482a0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ec98790, {0x51b9a20?, 0xc010b7a400}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc012448410 sp=0xc012448308 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc9?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc012448440 sp=0xc012448410 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc00bc43200, 0xc012448518, 0xc0124484e8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc012448498 sp=0xc012448440 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00bc43200, {0x51b9a20?, 0xc010b7a400?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc012448548 sp=0xc012448498 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc0129561e0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00a5aef00}, {0x51b9a20, 0xc010b7a400}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0124485b0 sp=0xc012448548 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0129561e0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a5aef00}, {0x51b9a20, 0xc010b7a400}, 0x6a8037abcfd82354?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0124486a0 sp=0xc0124485b0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x50cb41fc26ddb5df?, {0x624a948?, 0xc0129561e0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00a5aef00?}, {0x51b9a20?, 0xc010b7a400?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc012448740 sp=0xc0124486a0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123becb8, {0x624a948, 0xc0129561e0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0124487c0 sp=0xc012448740 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123becb8?}}, {0x624a948, 0xc0129561e0}, 0xc00a5aef00, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc012448898 sp=0xc0124487c0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x54900?, {0x624a948?, 0xc0129561e0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc0000549f8?}) :1 +0x65 fp=0xc0124488e8 sp=0xc012448898 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc018346060, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc012448b88 sp=0xc0124488e8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc018346060, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0f5e2b8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc012448cc0 sp=0xc012448b88 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0f5e2b8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc012449408 sp=0xc012448cc0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0f5e2b8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc012449d08 sp=0xc012449408 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x624a948, 0xc0129561e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1506 +0x1bc fp=0xc012449f30 sp=0xc012449d08 pc=0x19ca1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012449fe0 sp=0xc012449f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012449fe8 sp=0xc012449fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 457384 [sleep]: runtime.gopark(0x4b8aac82499?, 0x400e68?, 0x81?, 0x1e?, 0x80880de0ce?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002e09df0 sp=0xc002e09dd0 pc=0x4907d6 time.Sleep(0x139582448) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc002e09e30 sp=0xc002e09df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc004a74840, 0x400e68, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc002e09e88 sp=0xc002e09e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc004a74840, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc002e09ec8 sp=0xc002e09e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00679e000?, 0x32, 0xc005a50fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc002e09fa8 sp=0xc002e09ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc002e09fe0 sp=0xc002e09fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002e09fe8 sp=0xc002e09fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 488461 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x393?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00de2ef58 sp=0xc00de2ef38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7cc0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00de2ef90 sp=0xc00de2ef58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7bd8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00de2efb0 sp=0xc00de2ef90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc013da5b80?, 0xc00734e000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00de2efd8 sp=0xc00de2efb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc013da5b80, {0xc00734e000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00de2f058 sp=0xc00de2efd8 pc=0x535efa net.(*netFD).Read(0xc013da5b80, {0xc00734e000?, 0x7b5a68?, 0xc013da5b80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00de2f0a0 sp=0xc00de2f058 pc=0x7a20c9 net.(*conn).Read(0xc046762208, {0xc00734e000?, 0x164290f?, 0xc046762208?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00de2f0e8 sp=0xc00de2f0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc004fbb980, {0xc00734e000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00de2f130 sp=0xc00de2f0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc008004558, {0xc00734e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00de2f1d0 sp=0xc00de2f130 pc=0x370f896 bufio.(*Reader).fill(0xc00328c890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00de2f208 sp=0xc00de2f1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00328c890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00de2f228 sp=0xc00de2f208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00328ce68?, {0x62295c8, 0xc00328c890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00de2f260 sp=0xc00de2f228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00328c800, 0xc00d91a140, 0xc00de2f442, 0xc00de2f444, {0x623b718, 0xc0031746c0}, 0xc0059fa000, 0xc0031200a0, 0xc00de2f440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00de2f398 sp=0xc00de2f260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00328c800, {0x624a948, 0xc00dd37020}, 0xc00d91a120, 0xc0059fa000, 0xc00dd36f60, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00de2f768 sp=0xc00de2f398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc00dd37020}, {0x628a538, 0xc004fbb980}, {{{0xc007c7e038, 0x4}}, 0x1, 0xc00dd36f90, 0xc00dd36fc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00de2f8a8 sp=0xc00de2f768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc00dd37020}, {0x628a538?, 0xc004fbb980}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00de2fe20 sp=0xc00de2f8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00dd36d50?}, {0x628a538, 0xc004fbb980}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00de2feb0 sp=0xc00de2fe20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00dd36d50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00de2ff30 sp=0xc00de2feb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00de2ffe0 sp=0xc00de2ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00de2ffe8 sp=0xc00de2ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 436756 [sleep]: runtime.gopark(0x4b9299e329d?, 0x1206cc?, 0x50?, 0xd5?, 0x81138ff5ab?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00efc7df0 sp=0xc00efc7dd0 pc=0x4907d6 time.Sleep(0x146042da1) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00efc7e30 sp=0xc00efc7df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc02ec71620, 0x1206cc, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00efc7e88 sp=0xc00efc7e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc02ec71620, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00efc7ec8 sp=0xc00efc7e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00db41900?, 0x1a, 0xc007ca1f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00efc7fa8 sp=0xc00efc7ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00efc7fe0 sp=0xc00efc7fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00efc7fe8 sp=0xc00efc7fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 429080 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00b8a5640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0035f75d0 sp=0xc0035f75b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc011534fd0, 0x1e0) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0035f7618 sp=0xc0035f75d0 pc=0x4be30c sync.(*Cond).Wait(0xc004042800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0035f7650 sp=0xc0035f7618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc004042e68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc0035f7708 sp=0xc0035f7650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003bbec00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc0035f7cb8 sp=0xc0035f7708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003bbec00, {0x624a8a0, 0xc011535000}, 0x4115f7f000000000?, 0xc006385dc0?, 0xc011d956d0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0035f7d70 sp=0xc0035f7cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc011535000?}, {0xc0059fa000?}, 0x3?, 0xc004985ed8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0035f7e20 sp=0xc0035f7d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0035f7fe0 sp=0xc0035f7e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0035f7fe8 sp=0xc0035f7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 482930 [sleep]: runtime.gopark(0x4b927b7b877?, 0x1014bdd?, 0x73?, 0x86?, 0x811309a6c0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00be37df0 sp=0xc00be37dd0 pc=0x4907d6 time.Sleep(0x1476457f5) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00be37e30 sp=0xc00be37df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0030b7f20, 0x1014bdd, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00be37e88 sp=0xc00be37e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0030b7f20, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00be37ec8 sp=0xc00be37e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00b3c3f80?, {0xc00373a780?, 0x6, 0xc00b3c3f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00be37fa8 sp=0xc00be37ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00be37fe0 sp=0xc00be37fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00be37fe8 sp=0xc00be37fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 428276 [sleep]: runtime.gopark(0x4b8bf077d17?, 0x400e94?, 0x15?, 0x46?, 0x80a08cd263?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008869df0 sp=0xc008869dd0 pc=0x4907d6 time.Sleep(0x13d97bde2) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc008869e30 sp=0xc008869df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc03b5a5380, 0x400e94, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc008869e88 sp=0xc008869e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc03b5a5380, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc008869ec8 sp=0xc008869e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc001b42000?, 0x24, 0xc0079a2fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc008869fa8 sp=0xc008869ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc008869fe0 sp=0xc008869fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008869fe8 sp=0xc008869fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 414748 [select]: runtime.gopark(0xc015de1a90?, 0x4?, 0xd0?, 0x91?, 0xc015de19f8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc015de17a8 sp=0xc015de1788 pc=0x4907d6 runtime.selectgo(0xc015de1a90, 0xc015de19f0, 0x552aae3?, 0x0, 0xc015de1c80?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc015de18e8 sp=0xc015de17a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc00b56cf50, {0x624a948, 0xc010adc3f0}, {0xc014b51c20, {0x17600a76df557dee, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:153 +0x2f9 fp=0xc015de1fd0 sp=0xc015de18e8 pc=0x1ccedb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc010adc3f0}, 0xc004ef7540, 0xc015de1fe0?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc015de2440 sp=0xc015de1fd0 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc010adc3f0}, 0xc004ef7540?, {0xc014b51c20, {0x17600a76df557dee, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc015de2580 sp=0xc015de2440 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc010adc3f0}, 0xc004bb73b0, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc015de28a0 sp=0xc015de2580 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76df557dee, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc015de2c68 sp=0xc015de28a0 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76df557dee, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc015de35f0 sp=0xc015de2c68 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x624a948?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc015de3838 sp=0xc015de35f0 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc010adc390}, {0x203001?}, 0xc004bb72c0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc015de3ea8 sp=0xc015de3838 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc010adc300}, 0xc004bb72c0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc015de3f50 sp=0xc015de3ea8 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc010adc300?}, {0x52f0080?, 0xc004bb72c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc015de3f80 sp=0xc015de3f50 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc010adc300}, {0x52f0080, 0xc004bb72c0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc015de42b0 sp=0xc015de3f80 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc010adc300?}, {0x52f0080?, 0xc004bb72c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc015de42f0 sp=0xc015de42b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc010adc300}, {0x52f0080, 0xc004bb72c0}, 0xc015de4388?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc015de4320 sp=0xc015de42f0 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc010adc300?}, {0x52f0080?, 0xc004bb72c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc015de4360 sp=0xc015de4320 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc010adc300?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc015de4398 sp=0xc015de4360 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc010adc300}, {0xc00534d500?, 0x230f25c?}, 0xc015de4450) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc015de4410 sp=0xc015de4398 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc010adc300?}, {0x52f0080?, 0xc004bb72c0?}, 0x0?, 0x300?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc015de4488 sp=0xc015de4410 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc010adc300?}, {0x52f0080?, 0xc004bb72c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc015de44c8 sp=0xc015de4488 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc010adc300?}, {0xc010adc300?, 0x4?}, {0x52f0080?, 0xc004bb72c0?}, {0x51b9a20?, 0xc00ec17280}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc015de4508 sp=0xc015de44c8 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc010adc300}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc004bb72c0}, {0x51b9a20, 0xc00ec17280}, 0xc015de4680?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc015de45f8 sp=0xc015de4508 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc010adc300}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc004bb72c0}, {0x51b9a20, 0xc00ec17280}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc015de4690 sp=0xc015de45f8 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc010adc2a0}, 0xc004bb72c0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc015de4710 sp=0xc015de4690 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x532ec29?, {0x624a948?, 0xc010adc2a0?}, 0xc015de4808?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc015de4798 sp=0xc015de4710 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00d119320, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc015de4a38 sp=0xc015de4798 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00d119320, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc015de4b70 sp=0xc015de4a38 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc015de52b8 sp=0xc015de4b70 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc015de5bb8 sp=0xc015de52b8 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc015de6420 sp=0xc015de5bb8 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76df9e943f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc015de6840 sp=0xc015de6420 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00c994bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc015de6a98 sp=0xc015de6840 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00c994b80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc015de6bc0 sp=0xc015de6a98 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00c994b50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc015de6e50 sp=0xc015de6bc0 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00c994a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc015de73f0 sp=0xc015de6e50 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00c994a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc015de76e0 sp=0xc015de73f0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00c994920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc015de79e0 sp=0xc015de76e0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00c994900?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc015de7b88 sp=0xc015de79e0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00c994850, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc015de7da0 sp=0xc015de7b88 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00c994680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc015de7ff8 sp=0xc015de7da0 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc015de8120 sp=0xc015de7ff8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0064aa6e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc015de83e8 sp=0xc015de8120 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc015de8500 sp=0xc015de83e8 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00b57b980, {0x624a948, 0xc010adc1b0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc015de8720 sp=0xc015de8500 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00b57b980, {0x624a948, 0xc010adc1b0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc015de8a18 sp=0xc015de8720 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00d119da0, {0x624a948, 0xc010adc1b0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc015de8af8 sp=0xc015de8a18 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00c4e1500, {0x624a948, 0xc010adc1b0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc015de8dd0 sp=0xc015de8af8 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0108efea0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc015de8e28 sp=0xc015de8dd0 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc005f4e5c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc015de8e40 sp=0xc015de8e28 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc002bdb2c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc015de8e68 sp=0xc015de8e40 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc002bdb2c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc015de8e80 sp=0xc015de8e68 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc015de8e98 sp=0xc015de8e80 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x8?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc015de8ed8 sp=0xc015de8e98 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc002bdb2c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc015de8f10 sp=0xc015de8ed8 pc=0x29a100c github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*noopProcessor).Next(0xc00d3be500) github.com/cockroachdb/cockroach/pkg/sql/rowexec/noop.go:89 +0x46 fp=0xc015de8f48 sp=0xc015de8f10 pc=0x2595586 github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc00534d500, {{0x624a948?, 0xc010adc0c0?}, 0x1000000000005?, 0xc015de9048?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc015de8fc0 sp=0xc015de8f48 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc004241100, {{0x624a948, 0xc010adc0c0}, 0xc007b31c00, 0xc0042cb878}) github.com/cockroachdb/cockroach/pkg/sql/update.go:164 +0xc4 fp=0xc015de9020 sp=0xc015de8fc0 pc=0x352d844 github.com/cockroachdb/cockroach/pkg/sql.(*serializeNode).Next(0xc0271f1770, {{0x624a948?, 0xc010adc0c0?}, 0xc007b31c00?, 0xc0042cb878?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:115 +0x5a fp=0xc015de9058 sp=0xc015de9020 pc=0x3476ffa github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc00e272840) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc015de9150 sp=0xc015de9058 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc00cb50600) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:243 +0x13c fp=0xc015de94d0 sp=0xc015de9150 pc=0x299d6fc github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc0064ac840) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc015de94e8 sp=0xc015de94d0 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc015de9500 sp=0xc015de94e8 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc006df6a80?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc015de9540 sp=0xc015de9500 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc0064ac840, {0x624a8a0?, 0xc005f4e580?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc015de96d0 sp=0xc015de9540 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc004cd7540, {0x624a8a0, 0xc005f4e580}, 0xc0064aa6e0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc015de9770 sp=0xc015de96d0 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc0271f1830}, 0xc004bb6e10, 0xc0064aa6e0, 0xc00ec17080, 0xc00f612380, 0xc0042cbbb0, 0xc0468f8f00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc015dea020 sp=0xc015de9770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc0271f1830}, 0xe8?, 0xc004bb6e10, 0xc98d30ca834bbb01?, {{0x624ce40?, 0xc007912320?}, 0x0?}, 0xc00f612380, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc015dea0e0 sp=0xc015dea020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc0271f1830}, 0xc0042cbbb0, 0xc0042cb878?, 0xc0042cb878, 0xc00f612380, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc015dea1f8 sp=0xc015dea0e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0042cb300, {0x624a948?, 0xc0271f1830}, 0xc0042cb878, 0xc0271f1830?, {0x7f275f329680?, 0xc003579ac8}, 0x78?, 0xc02d810bb8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc015dea300 sp=0xc015dea1f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0042cb300, {0x624a8a0, 0xc005f4e000}, 0xc0042cb878, {0x7f275f329680, 0xc003579ac8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc015dea720 sp=0xc015dea300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0042cb300, {0x624a948?, 0xc012394450?}, {{0x6274800, 0xc004fb9880}, {0xc002ffd540, 0x7e}, 0x2, 0x1}, 0xc0115cd9e0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc015deb0b0 sp=0xc015dea720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc012394450?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc015deb168 sp=0xc015deb0b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc012394450?}, {0x6274800?, 0xc004fb9880?}, 0xc002ffd416?, 0xc0059fefa3?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc015deb2b8 sp=0xc015deb168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0042cb300, {0x624a948, 0xc012394450}, {{0x6274800, 0xc004fb9880}, {0xc002ffd540, 0x7e}, 0x2, 0x1}, 0xc0115cd9e0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc015deb430 sp=0xc015deb2b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0042cb300, {0x624a948, 0xc012394450}, {0xc0115cd9e0, {0xc013222160, 0x2, 0x2}, {0xc0056b24b0, 0x6, 0x6}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc015deb550 sp=0xc015deb430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0059fefe2, 0x0}, 0x0, {0xc1116e819277f2f9, 0x7fe8672146, 0x0}, 0x1}, 0xc0042cb300, 0xc015deb8c0, 0xc015deb8b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc015deb708 sp=0xc015deb550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0042cb300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc015debcb8 sp=0xc015deb708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0042cb300, {0x624a8a0, 0xc008c048c0}, 0x411a233800000000?, 0xc00168fdc0?, 0xc00d62ab50?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc015debd70 sp=0xc015debcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc008c048c0?}, {0xc0059fa000?}, 0x3?, 0xc00c10d410?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc015debe20 sp=0xc015debd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc015debfe0 sp=0xc015debe20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc015debfe8 sp=0xc015debfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 492242 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc0161ad640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0161ad5d0 sp=0xc0161ad5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc007fe6690, 0x37) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0161ad618 sp=0xc0161ad5d0 pc=0x4be30c sync.(*Cond).Wait(0xc008c88000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0161ad650 sp=0xc0161ad618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc008c88668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc0161ad708 sp=0xc0161ad650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc007c0cc00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc0161adcb8 sp=0xc0161ad708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc007c0cc00, {0x624a8a0, 0xc007fe66c0}, 0x41183c0400000000?, 0xc0056c7dc0?, 0xc0000aef70?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0161add70 sp=0xc0161adcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc007fe66c0?}, {0xc0059fa000?}, 0x3?, 0xc00322b380?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0161ade20 sp=0xc0161add70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0161adfe0 sp=0xc0161ade20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0161adfe8 sp=0xc0161adfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498365 [select]: runtime.gopark(0xc01014f2e8?, 0x3?, 0x4?, 0x0?, 0xc01014efb2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01014ed38 sp=0xc01014ed18 pc=0x4907d6 runtime.selectgo(0xc01014f2e8, 0xc01014efac, 0x17600a76492df4d4?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01014ee78 sp=0xc01014ed38 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc012b40d50}, 0xc00db28e10, 0xc0236061e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc01014f5e8 sp=0xc01014ee78 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc012b40d50}, 0xc00db28e10, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc01014f908 sp=0xc01014f5e8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76492df4d4, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc01014fcd0 sp=0xc01014f908 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76492df4d4, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc010150658 sp=0xc01014fcd0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0101508a0 sp=0xc010150658 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc012b40cf0}, {0x0?}, 0xc00db28d20) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc010150f10 sp=0xc0101508a0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc012b40c60}, 0xc00db28d20) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc010150fb8 sp=0xc010150f10 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc012b40c60?}, {0x52f0080?, 0xc00db28d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc010150fe8 sp=0xc010150fb8 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc012b40c60}, {0x52f0080, 0xc00db28d20}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc010151318 sp=0xc010150fe8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc012b40c60?}, {0x52f0080?, 0xc00db28d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010151358 sp=0xc010151318 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc012b40c60}, {0x52f0080, 0xc00db28d20}, 0xc00ba833f0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc010151388 sp=0xc010151358 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc012b40c60?}, {0x52f0080?, 0xc00db28d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0101513c8 sp=0xc010151388 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc012b40c60?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc010151400 sp=0xc0101513c8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc012b40c60}, {0x0?, 0x0?}, 0xc00ba834b8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc010151478 sp=0xc010151400 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc012b40c60?}, {0x52f0080?, 0xc00db28d20?}, 0x1?, 0x0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0101514f0 sp=0xc010151478 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc012b40c60?}, {0x52f0080?, 0xc00db28d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc010151530 sp=0xc0101514f0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc012b40c60?}, {0xc012b40c60?, 0x4?}, {0x52f0080?, 0xc00db28d20?}, {0x51b9a20?, 0xc004a54f80}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc010151570 sp=0xc010151530 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc012b40c60}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00db28d20}, {0x51b9a20, 0xc004a54f80}, 0xc00ba836e8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc010151660 sp=0xc010151570 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc012b40c60}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00db28d20}, {0x51b9a20, 0xc004a54f80}, 0xc00ba837c8?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc0101516f8 sp=0xc010151660 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc012b40ae0}, 0xc00db28d20, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc010151778 sp=0xc0101516f8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc0111721b0?, {0x624a948?, 0xc012b40ae0?}, 0xc00f615d40?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc010151800 sp=0xc010151778 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00df7af00, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010151aa0 sp=0xc010151800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00df7af00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010151bd8 sp=0xc010151aa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010152320 sp=0xc010151bd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010152c20 sp=0xc010152320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc010153488 sp=0xc010152c20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e200dcb5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0101538a8 sp=0xc010153488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0065cc0b8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc010153b00 sp=0xc0101538a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0065cc080, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc010153c28 sp=0xc010153b00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc012b40060}, {0x7f275f30db68, 0xc0065cc080}, 0xc00cc2afc0, {0xc0064b6000, 0x1e, 0x1f}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc010153e60 sp=0xc010153c28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc012b40060}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc010153f30 sp=0xc010153e60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010153fe0 sp=0xc010153f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010153fe8 sp=0xc010153fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 476831 [sleep]: runtime.gopark(0x4b8c2d43ae6?, 0x2061bd?, 0xb6?, 0x61?, 0x80acf7ee04?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a2c3df0 sp=0xc00a2c3dd0 pc=0x4907d6 time.Sleep(0x146361d16) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a2c3e30 sp=0xc00a2c3df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00c8418f0, 0x2061bd, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a2c3e88 sp=0xc00a2c3e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00c8418f0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a2c3ec8 sp=0xc00a2c3e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc001afc000?, 0x23, 0xc00e2e0f00?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a2c3fa8 sp=0xc00a2c3ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a2c3fe0 sp=0xc00a2c3fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a2c3fe8 sp=0xc00a2c3fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498498 [select]: runtime.gopark(0xc00a351c18?, 0x3?, 0xd2?, 0x19?, 0xc00a351b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a3518f0 sp=0xc00a3518d0 pc=0x4907d6 runtime.selectgo(0xc00a351c18, 0xc00a351b34, 0xc03bfb24c0?, 0x0, 0x7?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a351a30 sp=0xc00a3518f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc040031080}, {0x6221560, 0xc0065f9560}, {0x46?, {0xc00a351e01?, 0xa351e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00a351da0 sp=0xc00a351a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc040031080?}, 0x511, 0xc0065f9560, 0x1a?, 0xc011d94ee0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00a351e70 sp=0xc00a351da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc040031080}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00a351f30 sp=0xc00a351e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a351fe0 sp=0xc00a351f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a351fe8 sp=0xc00a351fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 466684 [sleep]: runtime.gopark(0x4b883630f1b?, 0x5a12f1?, 0x7d?, 0x7c?, 0x80473074ca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aa6df0 sp=0xc004aa6dd0 pc=0x4907d6 time.Sleep(0x11fdfce68) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004aa6e30 sp=0xc004aa6df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc03b5dc330, 0x5a12f1, {0x4618e7?, 0x20300e?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004aa6e88 sp=0xc004aa6e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc03b5dc330, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004aa6ec8 sp=0xc004aa6e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc014df1900?, 0xb, 0xc00c821f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004aa6fa8 sp=0xc004aa6ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004aa6fe0 sp=0xc004aa6fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aa6fe8 sp=0xc004aa6fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498325 [select]: runtime.gopark(0xc00c657208?, 0x2?, 0x98?, 0x8d?, 0xc00c657204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c657090 sp=0xc00c657070 pc=0x4907d6 runtime.selectgo(0xc00c657208, 0xc00c657200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c6571d0 sp=0xc00c657090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc015899e60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00c657238 sp=0xc00c6571d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0080513f0, {0x51b9a20?, 0xc03b0d1d00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00c657340 sp=0xc00c657238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00c657370 sp=0xc00c657340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0158998c0, 0xc00c657448, 0xc00c657418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00c6573c8 sp=0xc00c657370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0158998c0, {0x51b9a20?, 0xc03b0d1d00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00c657478 sp=0xc00c6573c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc010be2ed0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00495ae10}, {0x51b9a20, 0xc03b0d1d00}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00c6574e0 sp=0xc00c657478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc010be2ed0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00495ae10}, {0x51b9a20, 0xc03b0d1d00}, 0x7628b774a6bad9ac?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00c6575d0 sp=0xc00c6574e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc010be2ed0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00495ae10?}, {0x51b9a20?, 0xc03b0d1d00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00c657670 sp=0xc00c6575d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e35c8, {0x624a948, 0xc010be2ed0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00c6576f0 sp=0xc00c657670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e35c8?}}, {0x624a948, 0xc010be2ed0}, 0xc00495ae10, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00c6577c8 sp=0xc00c6576f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xb0fb830?, {0x624a948?, 0xc010be2ed0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00b0fb928?}) :1 +0x65 fp=0xc00c657818 sp=0xc00c6577c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0152e9620, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00c657ab8 sp=0xc00c657818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0152e9620, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de831d33, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00c657bf0 sp=0xc00c657ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de831d33, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00c658338 sp=0xc00c657bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de831d33, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00c658c38 sp=0xc00c658338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de831d33, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00c6594a0 sp=0xc00c658c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de831d33, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00c6598c0 sp=0xc00c6594a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00c6599f8 sp=0xc00c6598c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc030ec2f60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00c659c00 sp=0xc00c6599f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc030ec2f60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00c659f30 sp=0xc00c659c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c659fe0 sp=0xc00c659f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c659fe8 sp=0xc00c659fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 421152 [sleep]: runtime.gopark(0x4b85adadd0c?, 0x400e7e?, 0x7d?, 0xa?, 0x80202902cd?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aa5df0 sp=0xc004aa5dd0 pc=0x4907d6 time.Sleep(0x121608ef5) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004aa5e30 sp=0xc004aa5df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0060cc5a0, 0x400e7e, {0x4ccaee?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004aa5e88 sp=0xc004aa5e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0060cc5a0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004aa5ec8 sp=0xc004aa5e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc0029d8000?, 0x29, 0xc004ee7f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004aa5fa8 sp=0xc004aa5ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004aa5fe0 sp=0xc004aa5fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aa5fe8 sp=0xc004aa5fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 429792 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00ecc7640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ecc75d0 sp=0xc00ecc75b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc0069bb9d0, 0xe7) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00ecc7618 sp=0xc00ecc75d0 pc=0x4be30c sync.(*Cond).Wait(0xc00a6dd000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00ecc7650 sp=0xc00ecc7618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00a6dd668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc00ecc7708 sp=0xc00ecc7650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc007c08000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc00ecc7cb8 sp=0xc00ecc7708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc007c08000, {0x624a8a0, 0xc0069bba00}, 0x4119787000000000?, 0xc003a6bdc0?, 0xc0083ef110?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00ecc7d70 sp=0xc00ecc7cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0069bba00?}, {0xc0059fa000?}, 0x3?, 0xc002d79758?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00ecc7e20 sp=0xc00ecc7d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00ecc7fe0 sp=0xc00ecc7e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ecc7fe8 sp=0xc00ecc7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 440697 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x39c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e534f58 sp=0xc00e534f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7db0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00e534f90 sp=0xc00e534f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7cc8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00e534fb0 sp=0xc00e534f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00b141780?, 0xc007b97000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00e534fd8 sp=0xc00e534fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00b141780, {0xc007b97000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00e535058 sp=0xc00e534fd8 pc=0x535efa net.(*netFD).Read(0xc00b141780, {0xc007b97000?, 0x7b5a68?, 0xc00b141780?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00e5350a0 sp=0xc00e535058 pc=0x7a20c9 net.(*conn).Read(0xc0137e3c40, {0xc007b97000?, 0x164290f?, 0xc0137e3c40?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00e5350e8 sp=0xc00e5350a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00b3f6f00, {0xc007b97000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00e535130 sp=0xc00e5350e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01895cd68, {0xc007b97000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00e5351d0 sp=0xc00e535130 pc=0x370f896 bufio.(*Reader).fill(0xc00403f890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00e535208 sp=0xc00e5351d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00403f890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00e535228 sp=0xc00e535208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00403fe68?, {0x62295c8, 0xc00403f890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00e535260 sp=0xc00e535228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00403f800, 0xc00faa2aa0, 0xc00e535442, 0xc00e535444, {0x623b718, 0xc012426c60}, 0xc0059fa000, 0xc007a802b4, 0xc00e535440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00e535398 sp=0xc00e535260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00403f800, {0x624a948, 0xc04866dbf0}, 0xc00faa2a90, 0xc0059fa000, 0xc04866d740, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00e535768 sp=0xc00e535398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc04866dbf0}, {0x628a538, 0xc00b3f6f00}, {{{0xc006cc9038, 0x4}}, 0x1, 0xc04866d8c0, 0xc04866db90, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00e5358a8 sp=0xc00e535768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc04866dbf0}, {0x628a538?, 0xc00b3f6f00}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00e535e20 sp=0xc00e5358a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc04866d500?}, {0x628a538, 0xc00b3f6f00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00e535eb0 sp=0xc00e535e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc04866d500}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00e535f30 sp=0xc00e535eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00e535fe0 sp=0xc00e535f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e535fe8 sp=0xc00e535fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498414 [select]: runtime.gopark(0xc00fcf5108?, 0x5?, 0x0?, 0x0?, 0xc00fcf4fa6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fcf4de8 sp=0xc00fcf4dc8 pc=0x4907d6 runtime.selectgo(0xc00fcf5108, 0xc00fcf4f9c, 0x547b5eb?, 0x0, 0xc00fcf5028?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fcf4f28 sp=0xc00fcf4de8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc006f08000, {0x624a948, 0xc040030a20}, 0xc03080f2d8, 0x1, 0xc00fcf51b0?, 0x4e227c?, 0xc004e49740, 0xc013d532c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x254 fp=0xc00fcf5168 sp=0xc00fcf4f28 pc=0x1cb6cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x468005?, {0x624a948, 0xc040030a20}, 0xc0069f1238?, 0xc00fcf52b0, 0x1f?, 0x30?, 0xffffffffffffffff?, 0xc004e49740, 0x56840c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e fp=0xc00fcf51f0 sp=0xc00fcf5168 pc=0x1cb698e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc006f08000?, {0x624a948, 0xc040030a20}, 0xc004e49680, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc012aac7e0, 0x17}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 fp=0xc00fcf53d8 sp=0xc00fcf51f0 pc=0x1cb65f9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16?, {0x624a948, 0xc040030a20}, 0x6231758?, 0x467624a0?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x19d fp=0xc00fcf5520 sp=0xc00fcf53d8 pc=0x1cb4cbd github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc006ef86e0?, {0x624a948?, 0xc040030a20?}, {0xc004e49440, {0x17600a76a5521f84, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 fp=0xc00fcf5558 sp=0xc00fcf5520 pc=0x1cbf177 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc006ef86e0, {0x624a948, 0xc040030a20}, 0xc00d6bc140, 0xc00fcf5af8?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:275 +0x567 fp=0xc00fcf59c8 sp=0xc00fcf5558 pc=0x1cbd0e7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc040030a20}, 0xc00fcf5bb0?, {0xc004e49440, {0x17600a76a5521f84, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc00fcf5b08 sp=0xc00fcf59c8 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc040030a20}, 0xc010617a40, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc00fcf5e28 sp=0xc00fcf5b08 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76a5521f84, 0x0, 0x0}, 0x0, {0x17600a76e3e77f8e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00fcf61f0 sp=0xc00fcf5e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76a5521f84, 0x0, 0x0}, 0x0, {0x17600a76e3e77f8e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00fcf6b78 sp=0xc00fcf61f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3e77f8e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00fcf6dc0 sp=0xc00fcf6b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0400309c0}, {0xc00fcf7470?}, 0xc010617950) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00fcf7430 sp=0xc00fcf6dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0400308d0}, 0xc010617950) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00fcf74d8 sp=0xc00fcf7430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc0400308d0}, {0x52f0080?, 0xc010617950}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00fcf7518 sp=0xc00fcf74d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0400308d0}, {0x52f0080, 0xc010617950}, 0xc003402ba0, 0xc004feb398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00fcf7848 sp=0xc00fcf7518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0400308d0?}, {0x52f0080?, 0xc010617950?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00fcf7888 sp=0xc00fcf7848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0400308d0}, {0x52f0080, 0xc010617950}, 0xc00fcf7920?, 0xc003b9df80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00fcf78b8 sp=0xc00fcf7888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0400308d0?}, {0x52f0080?, 0xc010617950?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fcf78f8 sp=0xc00fcf78b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0400308d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00fcf7930 sp=0xc00fcf78f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0400308d0}, {0x203000?, 0x40?}, 0xc00fcf79e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00fcf79a8 sp=0xc00fcf7930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0400308d0?}, {0x52f0080?, 0xc010617950?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00fcf7a20 sp=0xc00fcf79a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0400308d0?}, {0x52f0080?, 0xc010617950?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fcf7a60 sp=0xc00fcf7a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc0400308d0}, {0x52f0080, 0xc010617950}, 0xc003402ba0, 0xc004feb398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00fcf7ab0 sp=0xc00fcf7a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc0400308d0}, 0xc011f07080, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00fcf7b08 sp=0xc00fcf7ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc003f74000, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00fcf7e48 sp=0xc00fcf7b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc003f74000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00fcf7f68 sp=0xc00fcf7e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00fcf7fe0 sp=0xc00fcf7f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fcf7fe8 sp=0xc00fcf7fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498482 [select]: runtime.gopark(0xc00885dc18?, 0x3?, 0xd2?, 0xd9?, 0xc00885db3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00885d8f0 sp=0xc00885d8d0 pc=0x4907d6 runtime.selectgo(0xc00885dc18, 0xc00885db34, 0xc00511cb80?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00885da30 sp=0xc00885d8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc00564c270}, {0x6221560, 0xc013848900}, {0x46?, {0xc00885de01?, 0x885de60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00885dda0 sp=0xc00885da30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc00564c270?}, 0x164, 0xc013848900, 0x1a?, 0xc000a714d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00885de70 sp=0xc00885dda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc00564c270}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00885df30 sp=0xc00885de70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00885dfe0 sp=0xc00885df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00885dfe8 sp=0xc00885dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498453 [select]: runtime.gopark(0xc00ef5b808?, 0x3?, 0x3?, 0x0?, 0xc00ef5b4d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ef5b258 sp=0xc00ef5b238 pc=0x4907d6 runtime.selectgo(0xc00ef5b808, 0xc00ef5b4cc, 0x17600a76c7de7cb4?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ef5b398 sp=0xc00ef5b258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc001f086f0}, 0xc0081da780, 0xc011371720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00ef5bb08 sp=0xc00ef5b398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc001f086f0}, 0xc0081da780, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00ef5be28 sp=0xc00ef5bb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76c5769900, 0x0, 0x0}, 0x0, {0x17600a76e1ce6894, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00ef5c1f0 sp=0xc00ef5be28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76c5769900, 0x0, 0x0}, 0x0, {0x17600a76e1ce6894, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00ef5cb78 sp=0xc00ef5c1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x8?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1ce6894, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00ef5cdc0 sp=0xc00ef5cb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc006b2f680}, {0x0?}, 0xc0081da4b0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00ef5d430 sp=0xc00ef5cdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc006b2e4b0}, 0xc0081da4b0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00ef5d4d8 sp=0xc00ef5d430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc006b2e4b0}, {0x52f0080?, 0xc0081da4b0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00ef5d518 sp=0xc00ef5d4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc006b2e4b0}, {0x52f0080, 0xc0081da4b0}, 0xc011dbb9c0, 0xc03d7ca078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00ef5d848 sp=0xc00ef5d518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc006b2e4b0?}, {0x52f0080?, 0xc0081da4b0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00ef5d888 sp=0xc00ef5d848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc006b2e4b0}, {0x52f0080, 0xc0081da4b0}, 0xc00ef5d920?, 0xc00a1d7fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00ef5d8b8 sp=0xc00ef5d888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc006b2e4b0?}, {0x52f0080?, 0xc0081da4b0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00ef5d8f8 sp=0xc00ef5d8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc006b2e4b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00ef5d930 sp=0xc00ef5d8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc006b2e4b0}, {0x203004?, 0x40?}, 0xc00ef5d9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00ef5d9a8 sp=0xc00ef5d930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc006b2e4b0?}, {0x52f0080?, 0xc0081da4b0?}, 0x5003b00?, 0x7f278cfaaa01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00ef5da20 sp=0xc00ef5d9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc006b2e4b0?}, {0x52f0080?, 0xc0081da4b0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00ef5da60 sp=0xc00ef5da20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc006b2e4b0}, {0x52f0080, 0xc0081da4b0}, 0xc011dbb9c0, 0xc03d7ca078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00ef5dab0 sp=0xc00ef5da60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc006b2e4b0}, 0xc00823a240, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00ef5db08 sp=0xc00ef5dab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc0106117a0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00ef5de48 sp=0xc00ef5db08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc0106117a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00ef5df68 sp=0xc00ef5de48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00ef5dfe0 sp=0xc00ef5df68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ef5dfe8 sp=0xc00ef5dfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 364986 [sleep]: runtime.gopark(0x4b88c0b1eeb?, 0x40bed0?, 0xc8?, 0xc0?, 0x80627f8328?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0039c7df0 sp=0xc0039c7dd0 pc=0x4907d6 time.Sleep(0x13286cd3e) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc0039c7e30 sp=0xc0039c7df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0147e54a0, 0x40bed0, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc0039c7e88 sp=0xc0039c7e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0147e54a0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc0039c7ec8 sp=0xc0039c7e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc004de6000?, 0x28, 0xc00da26f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc0039c7fa8 sp=0xc0039c7ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc0039c7fe0 sp=0xc0039c7fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0039c7fe8 sp=0xc0039c7fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 486522 [sleep]: runtime.gopark(0x4b880083919?, 0x80199f?, 0xe4?, 0x1d?, 0x8042b41641?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006644df0 sp=0xc006644dd0 pc=0x4907d6 time.Sleep(0x11ebe464e) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc006644e30 sp=0xc006644df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc013124060, 0x80199f, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc006644e88 sp=0xc006644e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc013124060, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc006644ec8 sp=0xc006644e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc011270500?, 0xa, 0xc00670ffb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc006644fa8 sp=0xc006644ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc006644fe0 sp=0xc006644fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006644fe8 sp=0xc006644fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 344854 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1f2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00be32f58 sp=0xc00be32f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbdac0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00be32f90 sp=0xc00be32f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd9d8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00be32fb0 sp=0xc00be32f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc015427080?, 0xc006157000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00be32fd8 sp=0xc00be32fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc015427080, {0xc006157000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00be33058 sp=0xc00be32fd8 pc=0x535efa net.(*netFD).Read(0xc015427080, {0xc006157000?, 0x7b5a68?, 0xc015427080?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00be330a0 sp=0xc00be33058 pc=0x7a20c9 net.(*conn).Read(0xc003731a50, {0xc006157000?, 0x164290f?, 0xc003731a50?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00be330e8 sp=0xc00be330a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01584aa80, {0xc006157000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00be33130 sp=0xc00be330e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc014ebc138, {0xc006157000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00be331d0 sp=0xc00be33130 pc=0x370f896 bufio.(*Reader).fill(0xc00a6da090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00be33208 sp=0xc00be331d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00a6da090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00be33228 sp=0xc00be33208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00a6da668?, {0x62295c8, 0xc00a6da090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00be33260 sp=0xc00be33228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00a6da000, 0xc012352df0, 0xc00be33442, 0xc00be33444, {0x623b718, 0xc02ea341b0}, 0xc0059fa000, 0xc016efb244, 0xc00be33440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00be33398 sp=0xc00be33260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00a6da000, {0x624a948, 0xc00fa36d20}, 0xc012352de0, 0xc0059fa000, 0xc00fa36c60, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00be33768 sp=0xc00be33398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc00fa36d20}, {0x628a538, 0xc01584aa80}, {{{0xc00db2601f, 0x4}}, 0x1, 0xc00fa36c90, 0xc00fa36cc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00be338a8 sp=0xc00be33768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc00fa36d20}, {0x628a538?, 0xc01584aa80}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00be33e20 sp=0xc00be338a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00fa36ba0?}, {0x628a538, 0xc01584aa80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00be33eb0 sp=0xc00be33e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00fa36ba0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00be33f30 sp=0xc00be33eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00be33fe0 sp=0xc00be33f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00be33fe8 sp=0xc00be33fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 478961 [sleep]: runtime.gopark(0x4b861f6c38e?, 0x801905?, 0x5d?, 0x7d?, 0x802b5875b2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006641df0 sp=0xc006641dd0 pc=0x4907d6 time.Sleep(0x125741d03) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc006641e30 sp=0xc006641df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc030818ba0, 0x801905, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc006641e88 sp=0xc006641e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc030818ba0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc006641ec8 sp=0xc006641e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00fa34280?, 0xa, 0xc00b3c4fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc006641fa8 sp=0xc006641ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc006641fe0 sp=0xc006641fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006641fe8 sp=0xc006641fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498390 [select]: runtime.gopark(0xc011827808?, 0x3?, 0x3?, 0x0?, 0xc0118274d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011827258 sp=0xc011827238 pc=0x4907d6 runtime.selectgo(0xc011827808, 0xc0118274cc, 0x17600a76732a4929?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc011827398 sp=0xc011827258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc0023b9a70}, 0xc0134f01e0, 0xc006c00be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc011827b08 sp=0xc011827398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc0023b9a70}, 0xc0134f01e0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc011827e28 sp=0xc011827b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a7672ef153f, 0x0, 0x0}, 0x0, {0x17600a76e06a57e1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc0118281f0 sp=0xc011827e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a7672ef153f, 0x0, 0x0}, 0x0, {0x17600a76e06a57e1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc011828b78 sp=0xc0118281f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x7171704761446864?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e06a57e1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc011828dc0 sp=0xc011828b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0023b9950}, {0x6b6b516a48616b58?}, 0xc0134f00f0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc011829430 sp=0xc011828dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0023b94d0}, 0xc0134f00f0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0118294d8 sp=0xc011829430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc0023b94d0}, {0x52f0080?, 0xc0134f00f0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc011829518 sp=0xc0118294d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0023b94d0}, {0x52f0080, 0xc0134f00f0}, 0xc02acfd220, 0xc001b01c80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc011829848 sp=0xc011829518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0023b94d0?}, {0x52f0080?, 0xc0134f00f0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc011829888 sp=0xc011829848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0023b94d0}, {0x52f0080, 0xc0134f00f0}, 0xc011829920?, 0xc00b400980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0118298b8 sp=0xc011829888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0023b94d0?}, {0x52f0080?, 0xc0134f00f0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0118298f8 sp=0xc0118298b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0023b94d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc011829930 sp=0xc0118298f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0023b94d0}, {0x20300a?, 0x40?}, 0xc0118299e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0118299a8 sp=0xc011829930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0023b94d0?}, {0x52f0080?, 0xc0134f00f0?}, 0x5003b00?, 0x7f278cfaa501?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc011829a20 sp=0xc0118299a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0023b94d0?}, {0x52f0080?, 0xc0134f00f0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc011829a60 sp=0xc011829a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc0023b94d0}, {0x52f0080, 0xc0134f00f0}, 0xc02acfd220, 0xc001b01c80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc011829ab0 sp=0xc011829a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc0023b94d0}, 0xc008428060, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc011829b08 sp=0xc011829ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc010bef320, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc011829e48 sp=0xc011829b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc010bef320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc011829f68 sp=0xc011829e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc011829fe0 sp=0xc011829f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011829fe8 sp=0xc011829fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 469673 [sleep]: runtime.gopark(0x4b9171405f8?, 0x400e91?, 0x60?, 0xec?, 0x80fca342ca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00daafdf0 sp=0xc00daafdd0 pc=0x4907d6 time.Sleep(0x141a1a550) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00daafe30 sp=0xc00daafdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0028e79b0, 0x400e91, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00daafe88 sp=0xc00daafe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0028e79b0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00daafec8 sp=0xc00daafe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc003548360?, {0xc001b0e000?, 0x32, 0x0?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00daaffa8 sp=0xc00daafec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00daaffe0 sp=0xc00daaffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00daaffe8 sp=0xc00daaffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 418839 [runnable]: runtime.gopark(0xae1b69f9b1e?, 0x0?, 0x60?, 0xc9?, 0xc006414d10?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc014ce38d8 sp=0xc014ce38b8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee72c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc014ce3940 sp=0xc014ce38d8 pc=0x4a17de sync.runtime_SemacquireMutex(0x45f3b6?, 0x40?, 0x527ed80?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc014ce3970 sp=0xc014ce3940 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee710, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc014ce39c8 sp=0xc014ce3970 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0xc007793cc0?, 0x0?, 0xc01c712600?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc014ce3a10 sp=0xc014ce39c8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004e6b340, {0x117c93b?, 0x7f25815247cc?}, 0x0, 0x0, 0xc006414d10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc014ce3b10 sp=0xc014ce3a10 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2727 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc01c712600, {0xc002c80a80, 0x1f, 0x40}, {0xc0012e6c40, 0x1f, 0x20}, 0x40?, 0x5d?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:847 +0xf8 fp=0xc014ce3b78 sp=0xc014ce3b10 pc=0x10c5078 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc006415bf0?, {0xc002c80a80?, 0x1?, 0x20?}, {0xc0012e6c40?, 0x10?, 0xc01664aec0?}, 0x1f?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc014ce3bc8 sp=0xc014ce3b78 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc006415bf0, {0xc002c80a80, 0x1f, 0x40}, {0xc0012e6c40, 0x1f, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc014ce3c18 sp=0xc014ce3bc8 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc006414f20, {0xc0012e6c40?, 0x0?, 0x0?}, 0x7f25835cae2c?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc014ce3d48 sp=0xc014ce3c18 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc006414f20, {0xc002c80a80?, 0x40?, 0xc014ce3db8?}, {0xc0012e6c40?, 0x113bd8b?, 0xc014ce3dd0?}, 0xd5?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc014ce3d88 sp=0xc014ce3d48 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc006414d88, {0xc002c80a80?, 0x0?, 0x20?}, {0xc0012e6c40, 0x1f, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc014ce3de0 sp=0xc014ce3d88 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc006414a00, {0xc0012e6c40, 0x1f, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc014ce3e78 sp=0xc014ce3de0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc008309b88, {{0xc007fdbb20, 0x1e, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc014ce3ec8 sp=0xc014ce3e78 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00b707680, {0x624a948, 0xc011b12e10}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc014ce3f30 sp=0xc014ce3ec8 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc011b12e10}, {0x62b4fe0, 0xc008309b88}, {0xc007fdbb20, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc014ce4500 sp=0xc014ce3f30 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc011b12e10}, {0x628f760?, 0xc008309b80?}, {0xc007fdbb20, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc014ce4680 sp=0xc014ce4500 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc011b12e10?}, {0x628f760?, 0xc008309b80?}, {0xc007fdbb20?, 0x1c?, 0x17?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc014ce4770 sp=0xc014ce4680 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc00dc771e0}, {{0x17600a76e4c54436, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_txn.go:71 +0x317 fp=0xc014ce4bb8 sp=0xc014ce4770 pc=0x1cfd357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e4c54436, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc014ce5448 sp=0xc014ce4bb8 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc010616960, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc014ce72b8 sp=0xc014ce5448 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc014ce7d28, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc014ce7928 sp=0xc014ce72b8 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc003861800, {0x624a948, 0xc011b12e10}, 0xc010616960, 0xc012b14dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc014ce85b8 sp=0xc014ce7928 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc011b12e10}, 0xc010616960, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc014ce88d8 sp=0xc014ce85b8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc014ce8ca0 sp=0xc014ce88d8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc014ce9628 sp=0xc014ce8ca0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc014ce9870 sp=0xc014ce9628 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc011b12c90}, {0x687553756a617971?}, 0xc010616870) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc014ce9ee0 sp=0xc014ce9870 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc011b12c00}, 0xc010616870) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc014ce9f88 sp=0xc014ce9ee0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc011b12c00?}, {0x52f0080?, 0xc010616870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc014ce9fb8 sp=0xc014ce9f88 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc011b12c00}, {0x52f0080, 0xc010616870}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc014cea2e8 sp=0xc014ce9fb8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011b12c00?}, {0x52f0080?, 0xc010616870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc014cea328 sp=0xc014cea2e8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc011b12c00}, {0x52f0080, 0xc010616870}, 0xc014cea3c0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc014cea358 sp=0xc014cea328 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011b12c00?}, {0x52f0080?, 0xc010616870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc014cea398 sp=0xc014cea358 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc011b12c00?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc014cea3d0 sp=0xc014cea398 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc011b12c00}, {0x4c6b634a4a586d6f?, 0x4453684963687158?}, 0xc014cea488) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc014cea448 sp=0xc014cea3d0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc011b12c00?}, {0x52f0080?, 0xc010616870?}, 0x5768414b55775367?, 0x436963467066754d?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc014cea4c0 sp=0xc014cea448 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011b12c00?}, {0x52f0080?, 0xc010616870?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc014cea500 sp=0xc014cea4c0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc011b12c00?}, {0xc011b12c00?, 0x4?}, {0x52f0080?, 0xc010616870?}, {0x51b9a20?, 0xc011420080}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc014cea540 sp=0xc014cea500 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011b12c00}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc010616870}, {0x51b9a20, 0xc011420080}, 0xc014cea6b8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc014cea630 sp=0xc014cea540 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc011b12c00}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc010616870}, {0x51b9a20, 0xc011420080}, 0xc014cea798?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc014cea6c8 sp=0xc014cea630 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc011b12ba0}, 0xc010616870, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc014cea748 sp=0xc014cea6c8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x736a586275425050?, {0x624a948?, 0xc011b12ba0?}, 0x77007a506b68514f?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc014cea7d0 sp=0xc014cea748 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0152e9e00, {_, _}, _, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc014ceaa70 sp=0xc014cea7d0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0152e9e00, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc014ceaba8 sp=0xc014ceaa70 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc014ceb2f0 sp=0xc014ceaba8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc014cebbf0 sp=0xc014ceb2f0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc014cec458 sp=0xc014cebbf0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x17600a76e4c56d72, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc014cec878 sp=0xc014cec458 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc014cec9b0 sp=0xc014cec878 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc014cecad8 sp=0xc014cec9b0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0x9c5fde0?, {_, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc014cecbf8 sp=0xc014cecad8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a8a0?, _}, {{{0x17600a76e4c54436, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc014cecd08 sp=0xc014cecbf8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a8a0, 0xc00511dcc0}, 0x0?, 0xc0078bc000) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc014cecf00 sp=0xc014cecd08 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a8a0, 0xc00511dcc0}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc014cecf48 sp=0xc014cecf00 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc004959320, {0x624a8a0, 0xc00511dcc0}, {{0xc7, 0x8f, 0xb7, 0x81, 0xb3, 0x9f, 0x44, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc014cecf98 sp=0xc014cecf48 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc004959320, {0x624a948, 0xc00564d710}, 0xc003553e00, 0xc010daa520, 0xc01d62cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:616 +0xcca fp=0xc014ced878 sp=0xc014cecf98 pc=0x19b974a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x624a948?, 0xc00564d710?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:515 +0x52 fp=0xc014ced8c8 sp=0xc014ced878 pc=0x19b8912 runtime/pprof.Do({0x624a948?, 0xc00564d680?}, {{0xc00511dc80?, 0x624a948?, 0xc00564d680?}}, 0xc014cedb70) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc014ced938 sp=0xc014ced8c8 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc004959320, {0x624a948, 0xc00564d680}, 0xc003553e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:514 +0xb25 fp=0xc014cedc00 sp=0xc014ced938 pc=0x19b87c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc00b56cf00, {0x624a948, 0xc00564d680}, {0x0, {0x17600a76e48a0920, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:348 +0x9f fp=0xc014cedc40 sp=0xc014cedc00 pc=0x1cbd8bf github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc00564d680}, 0xc002dda640, 0xc014cee1e0?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc014cee0b0 sp=0xc014cedc40 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00564d680}, 0xc014cee298?, {0x0, {0x17600a76e48a0920, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc014cee1f0 sp=0xc014cee0b0 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00564d680}, 0xc017226e10, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc014cee510 sp=0xc014cee1f0 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc014cee8d8 sp=0xc014cee510 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc014cef260 sp=0xc014cee8d8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc014cef4a8 sp=0xc014cef260 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00564d620}, {0x0?}, 0xc017226d20) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc014cefb18 sp=0xc014cef4a8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00564d530}, 0xc017226d20) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc014cefbc0 sp=0xc014cefb18 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00564d530?}, {0x52f0080?, 0xc017226d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc014cefbf0 sp=0xc014cefbc0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00564d530}, {0x52f0080, 0xc017226d20}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc014ceff20 sp=0xc014cefbf0 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d530?}, {0x52f0080?, 0xc017226d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc014ceff60 sp=0xc014ceff20 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00564d530}, {0x52f0080, 0xc017226d20}, 0xc014cefff8?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc014ceff90 sp=0xc014ceff60 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d530?}, {0x52f0080?, 0xc017226d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc014ceffd0 sp=0xc014ceff90 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00564d530?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc014cf0008 sp=0xc014ceffd0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00564d530}, {0x0?, 0x0?}, 0xc014cf00c0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc014cf0080 sp=0xc014cf0008 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00564d530?}, {0x52f0080?, 0xc017226d20?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc014cf00f8 sp=0xc014cf0080 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d530?}, {0x52f0080?, 0xc017226d20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc014cf0138 sp=0xc014cf00f8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00564d530?}, {0xc00564d530?, 0x4?}, {0x52f0080?, 0xc017226d20?}, {0x51b9a20?, 0xc0533f6180}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc014cf0178 sp=0xc014cf0138 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00564d530}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017226d20}, {0x51b9a20, 0xc0533f6180}, 0xc014cf02f0?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc014cf0268 sp=0xc014cf0178 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00564d530}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017226d20}, {0x51b9a20, 0xc0533f6180}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc014cf0300 sp=0xc014cf0268 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00564d4d0}, 0xc017226d20, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc014cf0380 sp=0xc014cf0300 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x0?, {0x624a948?, 0xc00564d4d0?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc014cf0408 sp=0xc014cf0380 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0152e99e0, {_, _}, _, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc014cf06a8 sp=0xc014cf0408 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0152e99e0, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc014cf07e0 sp=0xc014cf06a8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc014cf0f28 sp=0xc014cf07e0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc014cf1828 sp=0xc014cf0f28 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc014cf2090 sp=0xc014cf1828 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x17600a76e48a2d04, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc014cf24b0 sp=0xc014cf2090 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc014cf25e8 sp=0xc014cf24b0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc014cf2710 sp=0xc014cf25e8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0xc000ef2a80?, {_, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc014cf2830 sp=0xc014cf2710 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e48a0920, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc014cf2940 sp=0xc014cf2830 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc008334600}, 0x0?, 0xc0174fe580) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc014cf2b38 sp=0xc014cf2940 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc008334600}, 0x19?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc014cf2b80 sp=0xc014cf2b38 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0005593b0, {0x624a948, 0xc008334600}, 0xc014cf30f0, {{0x17600a76d6791b41, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:395 +0x6fb fp=0xc014cf2ff0 sp=0xc014cf2b80 pc=0x1ca4d3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0xc0174fd9e0?, {0x624a948, 0xc008334600}, 0xc0152471b0, {{0x17600a76d6791b41, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x145 fp=0xc014cf3200 sp=0xc014cf2ff0 pc=0x1ca4505 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc00b56cf50, {0x624a948, 0xc008334600}, {0xc0150147e0, {0x17600a76b8abb641, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:519 +0x596 fp=0xc014cf37b8 sp=0xc014cf3200 pc=0x1cd0d36 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc008334600?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc014cf38e8 sp=0xc014cf37b8 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc00b56cf50, {0x624a948, 0xc008334600}, {0xc0150147e0, {0x17600a76b8abb641, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc014cf3fd0 sp=0xc014cf38e8 pc=0x1ccefa6 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498435 [semacquire]: runtime.gopark(0xae1b7108153?, 0xc00a4778b0?, 0x60?, 0x3?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a477348 sp=0xc00a477328 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x34?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00a4773b0 sp=0xc00a477348 pc=0x4a17de sync.runtime_SemacquireMutex(0xc00a477470?, 0x20?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00a4773e0 sp=0xc00a4773b0 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x7f273cbcbde0?, 0xc0081f4cc0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00a477438 sp=0xc00a4773e0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc0107fc580?, 0xc00a4774c8?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00a477480 sp=0xc00a477438 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0107fc580, {0x17?, 0x17?}, 0x0, 0xc02d849d78, 0xc004c0e810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00a477580 sp=0xc00a477480 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc02d849b00, 0x70?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00a477620 sp=0xc00a477580 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc02d849b00, {0xc005e3eb70, 0x11, 0x11}, 0x20?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc00a4776d0 sp=0xc00a477620 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc0014cabe0?, 0x0?, 0x0?}, {0xc005e3eb70?, 0xf?, 0xc0069086e0?}, 0x30?, 0xff?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc00a477738 sp=0xc00a4776d0 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc004c0fde0?, {0xc0014cabe0?, 0x1?, 0x11?}, {0xc005e3eb70?, 0xc005e3eb70?, 0x0?}, 0x11?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00a477788 sp=0xc00a477738 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc004c0fde0, {0xc0014cabe0, 0x11, 0x20}, {0xc005e3eb70, 0x11, 0x11}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00a4777d8 sp=0xc00a477788 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc004c0ea20, {0xc005e3eb70?, 0x0?, 0x0?}, 0x0?, 0xed?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00a477908 sp=0xc00a4777d8 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc004c0ea20, {0xc0014cabe0?, 0x11?, 0xc0014cb100?}, {0xc005e3eb70?, 0x20?, 0x10?}, 0x10?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00a477948 sp=0xc00a477908 pc=0x115285d github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc004c0e500, {0xc005e3eb70, 0x11, 0x11}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00a4779e0 sp=0xc00a477948 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGE(0xc001d6f9b8, {{0xc00413ab70, 0x10, 0x10}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:347 +0xd3 fp=0xc00a477a38 sp=0xc00a4779e0 pc=0x1862f13 github.com/cockroachdb/cockroach/pkg/storage.GetIntent({0x628f760, 0xc001d6f700}, {0xc0054c7140, 0xa, 0x10}) github.com/cockroachdb/cockroach/pkg/storage/engine.go:1120 +0x1f6 fp=0xc00a477ec8 sp=0xc00a477a38 pc=0x1806d56 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryIntent({_, _}, {_, _}, {{0x62cc5e0, 0xc04384f1d0}, {{0x17600a765e48797d, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_intent.go:79 +0x2a6 fp=0xc00a478108 sp=0xc00a477ec8 pc=0x1cfabc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a765e48797d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc00a478998 sp=0xc00a478108 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc005422000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00a47a808 sp=0xc00a478998 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00a47b278, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc00a47ae78 sp=0xc00a47a808 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc00737e800, {0x624a948, 0xc018c52c90}, 0xc005422000, 0xc00ed13c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc00a47bb08 sp=0xc00a47ae78 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00737e800, {0x624a948, 0xc018c52c90}, 0xc005422000, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00a47be28 sp=0xc00a47bb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00737e800, {_, _}, {{{0x17600a765e48797d, 0x0, 0x0}, 0x0, {0x17600a76e16a1b06, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00a47c1f0 sp=0xc00a47be28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a765e48797d, 0x0, 0x0}, 0x0, {0x17600a76e16a1b06, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00a47cb78 sp=0xc00a47c1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e16a1b06, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00a47cdc0 sp=0xc00a47cb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc018c52b40}, {0xc00a47d470?}, 0xc0107d1ef0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00a47d430 sp=0xc00a47cdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc018c52a50}, 0xc0107d1ef0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00a47d4d8 sp=0xc00a47d430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc018c52a50}, {0x52f0080?, 0xc0107d1ef0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00a47d518 sp=0xc00a47d4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc018c52a50}, {0x52f0080, 0xc0107d1ef0}, 0xc010c703c0, 0xc013dccf00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00a47d848 sp=0xc00a47d518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c52a50?}, {0x52f0080?, 0xc0107d1ef0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00a47d888 sp=0xc00a47d848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc018c52a50}, {0x52f0080, 0xc0107d1ef0}, 0xc00a47d920?, 0xc008d8cb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00a47d8b8 sp=0xc00a47d888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c52a50?}, {0x52f0080?, 0xc0107d1ef0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00a47d8f8 sp=0xc00a47d8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc018c52a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00a47d930 sp=0xc00a47d8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc018c52a50}, {0x203004?, 0x40?}, 0xc00a47d9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00a47d9a8 sp=0xc00a47d930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc018c52a50?}, {0x52f0080?, 0xc0107d1ef0?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00a47da20 sp=0xc00a47d9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c52a50?}, {0x52f0080?, 0xc0107d1ef0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00a47da60 sp=0xc00a47da20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc018c52a50}, {0x52f0080, 0xc0107d1ef0}, 0xc010c703c0, 0xc013dccf00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00a47dab0 sp=0xc00a47da60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc018c52a50}, 0xc00d1de780, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00a47db08 sp=0xc00a47dab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc008148ea0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00a47de48 sp=0xc00a47db08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc008148ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00a47df68 sp=0xc00a47de48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00a47dfe0 sp=0xc00a47df68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a47dfe8 sp=0xc00a47dfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 439885 [sleep]: runtime.gopark(0x4b84f6840d0?, 0x1002fa5?, 0x97?, 0x92?, 0x801093c0ea?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006640df0 sp=0xc006640dd0 pc=0x4907d6 time.Sleep(0x11d3de9c1) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc006640e30 sp=0xc006640df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc004773ec0, 0x1002fa5, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc006640e88 sp=0xc006640e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc004773ec0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc006640ec8 sp=0xc006640e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc001405f80?, {0xc004a22000?, 0x12, 0xc001405f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc006640fa8 sp=0xc006640ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc006640fe0 sp=0xc006640fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006640fe8 sp=0xc006640fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498457 [select]: runtime.gopark(0xc00b79b208?, 0x2?, 0x10?, 0xa2?, 0xc00b79b204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b79b090 sp=0xc00b79b070 pc=0x4907d6 runtime.selectgo(0xc00b79b208, 0xc00b79b200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b79b1d0 sp=0xc00b79b090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc013d53d40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00b79b238 sp=0xc00b79b1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018ac2c0, {0x51b9a20?, 0xc004a55d00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00b79b340 sp=0xc00b79b238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00b79b370 sp=0xc00b79b340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc013d53b00, 0xc00b79b448, 0xc00b79b418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00b79b3c8 sp=0xc00b79b370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc013d53b00, {0x51b9a20?, 0xc004a55d00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00b79b478 sp=0xc00b79b3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc001f09380?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0081da870}, {0x51b9a20, 0xc004a55d00}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00b79b4e0 sp=0xc00b79b478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc001f09380}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081da870}, {0x51b9a20, 0xc004a55d00}, 0x55fd214fa6420045?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00b79b5d0 sp=0xc00b79b4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc001f09380?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0081da870?}, {0x51b9a20?, 0xc004a55d00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00b79b670 sp=0xc00b79b5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc012277248, {0x624a948, 0xc001f09380}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00b79b6f0 sp=0xc00b79b670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc012277248?}}, {0x624a948, 0xc001f09380}, 0xc0081da870, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00b79b7c8 sp=0xc00b79b6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xba83830?, {0x624a948?, 0xc001f09380?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00ba83928?}) :1 +0x65 fp=0xc00b79b818 sp=0xc00b79b7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0116b5680, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00b79bab8 sp=0xc00b79b818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0116b5680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e26d5664, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00b79bbf0 sp=0xc00b79bab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e26d5664, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00b79c338 sp=0xc00b79bbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e26d5664, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00b79cc38 sp=0xc00b79c338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e26d5664, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00b79d4a0 sp=0xc00b79cc38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e26d5664, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00b79d8c0 sp=0xc00b79d4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00b79d9f8 sp=0xc00b79d8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc001f09320}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00b79dc00 sp=0xc00b79d9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc001f09320}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00b79df30 sp=0xc00b79dc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b79dfe0 sp=0xc00b79df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b79dfe8 sp=0xc00b79dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 339455 [chan receive]: runtime.gopark(0x2faf080?, 0x3b9aca00?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010a1a508 sp=0xc010a1a4e8 pc=0x4907d6 runtime.chanrecv(0xc001a13200, 0xc010a1a750, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc010a1a598 sp=0xc010a1a508 pc=0x45a91b runtime.chanrecv1(0x1?, 0x7f275a1a1730?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc010a1a5c0 sp=0xc010a1a598 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1290 +0x127 fp=0xc010a1a788 sp=0xc010a1a5c0 pc=0x19c9a07 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0af9890, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1468 +0x17b9 fp=0xc010a1aff0 sp=0xc010a1a788 pc=0x19c93d9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0af9890, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc010a1b410 sp=0xc010a1aff0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc024bdebb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc010a1b668 sp=0xc010a1b410 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc024bdeb80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc010a1b790 sp=0xc010a1b668 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc024bdeb50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc010a1ba20 sp=0xc010a1b790 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc024bdea50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc010a1bfc0 sp=0xc010a1ba20 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc024bdea50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc010a1c2b0 sp=0xc010a1bfc0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc024bde920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc010a1c5b0 sp=0xc010a1c2b0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc024bde900?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc010a1c758 sp=0xc010a1c5b0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc024bde850, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc010a1c970 sp=0xc010a1c758 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc024bde680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc010a1cbc8 sp=0xc010a1c970 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc010a1ccf0 sp=0xc010a1cbc8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc02b24c8f0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc010a1cfb8 sp=0xc010a1ccf0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc010a1d0c8 sp=0xc010a1cfb8 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc0289f8810}, 0x456936?, 0xc01616f180) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc010a1d2c0 sp=0xc010a1d0c8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc02b24c8f0, {0x624a948, 0xc0289f8810}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc010a1d308 sp=0xc010a1d2c0 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc0016b2028, {0x624a948, 0xc0289f8810}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc010a1d340 sp=0xc010a1d308 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc0016b2000, {{0x624a948, 0xc0289f8810}, 0xc0065c8000, 0xc004177878}) github.com/cockroachdb/cockroach/pkg/sql/update.go:197 +0x225 fp=0xc010a1d3a0 sp=0xc010a1d340 pc=0x352d9a5 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc002e039f8, {{0x624a948?, 0xc0289f8810?}, 0xc0065c8000?, 0xc004177878?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc010a1d3f8 sp=0xc010a1d3a0 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc014aeb5f8?, 0x624d578?}, {0x624ceb0, 0xc002e039f8}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc010a1d4a8 sp=0xc010a1d3f8 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc010a1d4e0 sp=0xc010a1d4a8 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc014aeb5f8, {0x624ceb0?, 0xc002e039f8?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc010a1d570 sp=0xc010a1d4e0 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc014aeb5f8, {0x624ceb0, 0xc002e039f8}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc010a1d5c0 sp=0xc010a1d570 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc0289f8810?}, 0xc0065c8000?, 0xc004177878?}, {0x624ceb0, 0xc002e039f8}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc010a1d640 sp=0xc010a1d5c0 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc0111ce2c0, {0x624a948, 0xc0289f87e0}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc010a1d680 sp=0xc010a1d640 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00bb14600, {0x624a948, 0xc0289f87b0}) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:187 +0xea fp=0xc010a1d6b8 sp=0xc010a1d680 pc=0x299d40a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:247 +0x2e fp=0xc010a1d6e0 sp=0xc010a1d6b8 pc=0x322578e github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc00409a380?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc010a1d720 sp=0xc010a1d6e0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init(0xc0152b9c00?, {0x624a948?, 0xc0289f87b0?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:246 +0x4f fp=0xc010a1d758 sp=0xc010a1d720 pc=0x322572f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc00545fc30, {0x624a8a0?, 0xc01388e080?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:291 +0x28d fp=0xc010a1d8e8 sp=0xc010a1d758 pc=0x3225b8d github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0102440a0, {0x624a8a0, 0xc01388e080}, 0xc02b24c8f0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc010a1d988 sp=0xc010a1d8e8 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc01612dd10}, 0xc0120e2000, 0xc02b24c8f0, 0xc011cdd780, 0xc00534b500, 0xc004177bb0, 0xc00771b570) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc010a1e238 sp=0xc010a1d988 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc01612dd10}, 0xe8?, 0xc0120e2000, 0x5fd2d8e361e8b901?, {{0x624ceb0?, 0xc002e039f8?}, 0x0?}, 0xc00534b500, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc010a1e2f8 sp=0xc010a1e238 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc01612dd10}, 0xc004177bb0, 0xc004177878?, 0xc004177878, 0xc00534b500, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc010a1e410 sp=0xc010a1e2f8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc004177300, {0x624a948?, 0xc01612dd10}, 0xc004177878, 0xc01612dd10?, {0x7f275f329680?, 0xc0034bb6c8}, 0x3e5?, 0xc00b6dd8d8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc010a1e518 sp=0xc010a1e410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc004177300, {0x624a8a0, 0xc00ee8b500}, 0xc004177878, {0x7f275f329680, 0xc0034bb6c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc010a1e938 sp=0xc010a1e518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc004177300, {0x624a948?, 0xc00368dcb0?}, {{0x6274800, 0xc011cdce00}, {0xc003b0b861, 0x4fa}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc010a1f2c8 sp=0xc010a1e938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc00368dcb0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc010a1f380 sp=0xc010a1f2c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc00368dcb0?}, {0x6274800?, 0xc011cdce00?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc010a1f4d0 sp=0xc010a1f380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc004177300, {0x624a948, 0xc00368dcb0}, {{0x6274800, 0xc011cdce00}, {0xc003b0b861, 0x4fa}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc010a1f648 sp=0xc010a1f4d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6274800, 0xc011cdce00}, {0xc003b0b861, 0x4fa}, 0x0, 0x1}, {0xc1116e8192128965, 0x7fe801b7c8, 0x0}, {0xc1116e81921298a1, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc010a1f708 sp=0xc010a1f648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc004177300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc010a1fcb8 sp=0xc010a1f708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc004177300, {0x624a8a0, 0xc00502c300}, 0x4169dd8320000000?, 0xc0174c3dc0?, 0xc003451ae0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc010a1fd70 sp=0xc010a1fcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00502c300?}, {0xc0059fa000?}, 0x3?, 0xc006edbe78?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc010a1fe20 sp=0xc010a1fd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc010a1ffe0 sp=0xc010a1fe20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010a1ffe8 sp=0xc010a1ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498042 [select]: runtime.gopark(0xc0085c9c20?, 0x3?, 0x22?, 0x9b?, 0xc0085c9bda?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0085c9a40 sp=0xc0085c9a20 pc=0x4907d6 runtime.selectgo(0xc0085c9c20, 0xc0085c9bd4, 0xc014b67380?, 0x0, 0xc00fec07d0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0085c9b80 sp=0xc0085c9a40 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc005b2af20, {0x624a910, 0xc014b67380}, 0xc0085c9ca8?, {0x629caa8, 0xc002a88580}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x165 fp=0xc0085c9c60 sp=0xc0085c9b80 pc=0x1ca0605 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0005593b0, {0x624a910, 0xc014b67380}, 0xc0085c9dd0?, 0xc05a044480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:749 +0x3d1 fp=0xc0085c9da8 sp=0xc0085c9c60 pc=0x1ca75d1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x624a910?, 0xc014b67380?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:792 +0x38 fp=0xc0085c9de0 sp=0xc0085c9da8 pc=0x1ca7bd8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x624a948?, 0xc0196d7bc0?}, {0x537e81f, 0x12}, 0x4a817c800, 0xc0085c9f00) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed fp=0xc0085c9e80 sp=0xc0085c9de0 pc=0x16210cd github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x624a948, 0xc0196d7bc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:790 +0xac fp=0xc0085c9f30 sp=0xc0085c9e80 pc=0x1ca7aac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0085c9fe0 sp=0xc0085c9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0085c9fe8 sp=0xc0085c9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498157 [runnable]: github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl.(*cTRBlockCipherStream).transform(0xc0072dd050?, 0x8d6a?, {0x7f26f321ade4, 0x10, 0x12a73?}, {0xc0159a6a70?, 0x10?, 0x10?}) github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/ctr_stream.go:189 +0x13a fp=0xc00871e740 sp=0xc00871e738 pc=0x44530ba github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl.(*fileCipherStream).Encrypt(0xc043276fb8, 0x800dc, {0x7f26f320d820, 0x20037, 0x20037}) github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/ctr_stream.go:143 +0x20e fp=0xc00871e7d0 sp=0xc00871e740 pc=0x4452c0e github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl.(*fileCipherStream).Decrypt(0x10a3274?, 0x7f26f320d820?, {0x7f26f320d820?, 0x7f26f320d800?, 0xc00871e850?}) github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/ctr_stream.go:152 +0x1e fp=0xc00871e808 sp=0xc00871e7d0 pc=0x4452cde github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl.(*encryptedFile).ReadAt(0xc008d8dec0, {0x7f26f320d820, 0xc01278c01d?, 0x20037}, 0x3ffe3?) github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/encrypted_fs.go:102 +0x85 fp=0xc00871e860 sp=0xc00871e808 pc=0x4453585 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc008996580, {0x12?, 0x12?}, 0x0, 0xc007bf2878, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2811 +0x3b6 fp=0xc00871e960 sp=0xc00871e860 pc=0x10cb216 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc007bf2600, 0x10?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00871ea00 sp=0xc00871e960 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*compactionIterator).skipForward(0xc013dcdb48, 0xc0158b0000?, {0x0?, 0xc00871eab0?, 0x18?}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1391 +0x7f fp=0xc00871ea68 sp=0xc00871ea00 pc=0x10c6edf github.com/cockroachdb/pebble/sstable.(*compactionIterator).Next(0xc013dcdb48) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1378 +0x5d fp=0xc00871eaa0 sp=0xc00871ea68 pc=0x10c6dfd github.com/cockroachdb/pebble.(*levelIter).Next(0xc00895f180) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:806 +0x167 fp=0xc00871eac8 sp=0xc00871eaa0 pc=0x114c207 github.com/cockroachdb/pebble.(*mergingIter).nextEntry(0xc004fcc700, 0xc014f09600) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:555 +0x6c fp=0xc00871eb18 sp=0xc00871eac8 pc=0x1150d8c github.com/cockroachdb/pebble.(*mergingIter).Next(0xc004fcc700) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:1107 +0x4e fp=0xc00871eb38 sp=0xc00871eb18 pc=0x115326e github.com/cockroachdb/pebble.(*compactionIter).iterNext(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:473 github.com/cockroachdb/pebble.(*compactionIter).nextInStripe(0xc006e49980) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:496 +0x2e fp=0xc00871eb88 sp=0xc00871eb38 pc=0x110df4e github.com/cockroachdb/pebble.(*compactionIter).setNext(0xc006e49980) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:574 +0x27f fp=0xc00871ebe0 sp=0xc00871eb88 pc=0x110e43f github.com/cockroachdb/pebble.(*compactionIter).Next(0xc006e49980) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:368 +0x705 fp=0xc00871ec80 sp=0xc00871ebe0 pc=0x110d9a5 github.com/cockroachdb/pebble.(*DB).runCompaction(0xc002a2cd80, 0x1329, 0xc00486d200) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2690 +0x1ea5 fp=0xc00871fc60 sp=0xc00871ec80 pc=0x1107205 github.com/cockroachdb/pebble.(*DB).compact1(0xc002a2cd80, 0xc00486d200, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2180 +0x1a5 fp=0xc00871fe80 sp=0xc00871fc60 pc=0x1104905 github.com/cockroachdb/pebble.(*DB).compact.func1({0x624a948, 0xc044dfe750}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2151 +0xad fp=0xc00871fee8 sp=0xc00871fe80 pc=0x110468d runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc000084440?, 0x1085cdfb0?, 0xc00d245750?}}, 0xc0085cdf88) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00871ff58 sp=0xc00871fee8 pc=0xfd3223 github.com/cockroachdb/pebble.(*DB).compact(0xc0085cdfd0?, 0x10d7606?, 0xc00d245740?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2148 +0x6b fp=0xc00871ffb8 sp=0xc00871ff58 pc=0x11045ab github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker.func4() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1869 +0x2c fp=0xc00871ffe0 sp=0xc00871ffb8 pc=0x1102b0c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00871ffe8 sp=0xc00871ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1869 +0x5ea goroutine 497360 [runnable]: runtime.gopark(0xae1b6f41594?, 0xae170d623d4?, 0xe0?, 0x16?, 0xc00c3471a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01dab6380 sp=0xc01dab6360 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee7fc, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc01dab63e8 sp=0xc01dab6380 pc=0x4a17de sync.runtime_SemacquireMutex(0xc0005ee3e0?, 0x78?, 0x46b793?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc01dab6418 sp=0xc01dab63e8 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee7e0, 0x0?, 0xc01dab64d8?, 0x487350?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc01dab6470 sp=0xc01dab6418 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x2000?, 0x0?, 0x0?, 0x3c?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc01dab64b8 sp=0xc01dab6470 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0107fc580, {0xc01dab65f8?, 0x4ccece?}, 0x0, 0x0, 0xc00671e310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc01dab65b8 sp=0xc01dab64b8 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc01839af00, 0xc0107fc580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc01dab6608 sp=0xc01dab65b8 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc0107fc580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc00671e310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc01dab6690 sp=0xc01dab6608 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc0029416b0, 0xb8f504?, 0xc00671f930, {0x0?, 0xc00671e310?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc01dab67c0 sp=0xc01dab6690 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x12?, 0x0?, 0xc007412640?, {0x0?, 0xc00671e310?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc01dab6810 sp=0xc01dab67c0 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc00671f8e0?, 0xc0069086e0?, {0x0?, 0xc00671e310?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc01dab6858 sp=0xc01dab6810 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc00671f8e0, 0xc0069086e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc01dab68b8 sp=0xc01dab6858 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00671f8e0, {0xc007412640, 0x20, 0x50}, {0xc00815bee0, 0x20, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc01dab6908 sp=0xc01dab68b8 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00671e520, {0xc00815bee0?, 0x0?, 0x1000000000000?}, 0x7f275f295d88?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc01dab6a38 sp=0xc01dab6908 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00671e520, {0xc007412640?, 0x0?, 0xc01dab6aa8?}, {0xc00815bee0?, 0x4a59d2?, 0x20?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc01dab6a78 sp=0xc01dab6a38 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc00671e388, {0xc007412640?, 0x0?, 0x20?}, {0xc00815bee0, 0x20, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc01dab6ad0 sp=0xc01dab6a78 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00671e000, {0xc00815bee0, 0x20, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc01dab6b68 sp=0xc01dab6ad0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0000dd708, {{0xc010482450, 0x1f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc01dab6bb8 sp=0xc01dab6b68 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00d07b680, {0x624a948, 0xc005756a50}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc01dab6c20 sp=0xc01dab6bb8 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc005756a50}, {0x62b4fe0, 0xc0000dd708}, {0xc010482450, 0x1f, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc01dab71f0 sp=0xc01dab6c20 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc005756a50}, {0x628f760?, 0xc0000dd700?}, {0xc010482450, 0x1f, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc01dab7370 sp=0xc01dab71f0 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc005756a50?}, {0x628f760?, 0xc0000dd700?}, {0xc010482450?, 0x19?, 0x1d?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc01dab7460 sp=0xc01dab7370 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc00dc77e40}, {{0x17600a76e06d541d, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_txn.go:71 +0x317 fp=0xc01dab78a8 sp=0xc01dab7460 pc=0x1cfd357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e06d541d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc01dab8138 sp=0xc01dab78a8 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc004e9b680, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc01dab9fa8 sp=0xc01dab8138 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc01dabaa18, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc01daba618 sp=0xc01dab9fa8 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc006f10000, {0x624a948, 0xc005756a50}, 0xc004e9b680, 0xc00185ce60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc01dabb2a8 sp=0xc01daba618 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc005756a50}, 0xc004e9b680, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc01dabb5c8 sp=0xc01dabb2a8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc01dabb990 sp=0xc01dabb5c8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc01dabc318 sp=0xc01dabb990 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc01dabc560 sp=0xc01dabc318 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0057569f0}, {0x0?}, 0xc004e9b590) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc01dabcbd0 sp=0xc01dabc560 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc005756960}, 0xc004e9b590) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc01dabcc78 sp=0xc01dabcbd0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc005756960?}, {0x52f0080?, 0xc004e9b590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc01dabcca8 sp=0xc01dabcc78 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc005756960}, {0x52f0080, 0xc004e9b590}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc01dabcfd8 sp=0xc01dabcca8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc005756960?}, {0x52f0080?, 0xc004e9b590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc01dabd018 sp=0xc01dabcfd8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc005756960}, {0x52f0080, 0xc004e9b590}, 0xc01dabd0b0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc01dabd048 sp=0xc01dabd018 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc005756960?}, {0x52f0080?, 0xc004e9b590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc01dabd088 sp=0xc01dabd048 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc005756960?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc01dabd0c0 sp=0xc01dabd088 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc005756960}, {0x0?, 0x0?}, 0xc01dabd178) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc01dabd138 sp=0xc01dabd0c0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc005756960?}, {0x52f0080?, 0xc004e9b590?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc01dabd1b0 sp=0xc01dabd138 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc005756960?}, {0x52f0080?, 0xc004e9b590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc01dabd1f0 sp=0xc01dabd1b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc005756960?}, {0xc005756960?, 0x4?}, {0x52f0080?, 0xc004e9b590?}, {0x51b9a20?, 0xc0178ae780}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc01dabd230 sp=0xc01dabd1f0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc005756960}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc004e9b590}, {0x51b9a20, 0xc0178ae780}, 0xc01dabd3a8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc01dabd320 sp=0xc01dabd230 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc005756960}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc004e9b590}, {0x51b9a20, 0xc0178ae780}, 0xc01dabd488?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc01dabd3b8 sp=0xc01dabd320 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc005756900}, 0xc004e9b590, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc01dabd438 sp=0xc01dabd3b8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x51090a0?, {0x624a948?, 0xc005756900?}, 0xc00d2ede80?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc01dabd4c0 sp=0xc01dabd438 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc012a93b00, {_, _}, _, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01dabd760 sp=0xc01dabd4c0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc012a93b00, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01dabd898 sp=0xc01dabd760 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01dabdfe0 sp=0xc01dabd898 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01dabe8e0 sp=0xc01dabdfe0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc01dabf148 sp=0xc01dabe8e0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x17600a76e06d6235, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01dabf568 sp=0xc01dabf148 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc01dabf6a0 sp=0xc01dabf568 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc01dabf7c8 sp=0xc01dabf6a0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0xc01dabfaa8?, {_, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc01dabf8e8 sp=0xc01dabf7c8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e06d541d, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc01dabf9f8 sp=0xc01dabf8e8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc00e7e6090}, 0x0?, 0xc00a8ebb80) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc01dabfbf0 sp=0xc01dabf9f8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc00e7e6090}, 0x4596bd?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc01dabfc38 sp=0xc01dabfbf0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc006eacb40, {0x624a948, 0xc00e7e6090}, {{0x8d, 0x39, 0x4b, 0x4b, 0x47, 0x54, 0x4a, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc01dabfc88 sp=0xc01dabfc38 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).startQueryPusherTxn.func1({0x624a948, 0xc00e7e6090}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:854 +0x238 fp=0xc01dabff30 sp=0xc01dabfc88 pc=0x19bbe78 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01dabffe0 sp=0xc01dabff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01dabffe8 sp=0xc01dabffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498165 [semacquire]: runtime.gopark(0xae1b779ca47?, 0x0?, 0x40?, 0x68?, 0x6958c8af?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bb0c6a8 sp=0xc00bb0c688 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee728, 0x7?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00bb0c710 sp=0xc00bb0c6a8 pc=0x4a17de sync.runtime_SemacquireMutex(0x250?, 0x80?, 0x7f274218cae0?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00bb0c740 sp=0xc00bb0c710 pc=0x4be125 sync.(*RWMutex).Lock(0x740cb0?) GOROOT/src/sync/rwmutex.go:152 +0x71 fp=0xc00bb0c770 sp=0xc00bb0c740 pc=0x4cded1 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc0005ee710, 0x7f274216e2a0?, 0x7f274216e2a0?, 0x7f274216e280?, 0x7f274216e280) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 fp=0xc00bb0c808 sp=0xc00bb0c770 pc=0x10a3725 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc00c984440?, 0x7f274216e2a0?, 0x7f274216e2a0?, 0x20038?, 0x468005?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 fp=0xc00bb0c860 sp=0xc00bb0c808 pc=0x10a57a6 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc005c75600, {0x13?, 0x13?}, 0x0, 0xc008225778, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2854 +0x60f fp=0xc00bb0c960 sp=0xc00bb0c860 pc=0x10cb46f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc008225500, 0x11?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00bb0ca00 sp=0xc00bb0c960 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*compactionIterator).skipForward(0xc0057f4528, 0xc015baa000?, {0x0?, 0xc00bb0cab0?, 0x19?}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1391 +0x7f fp=0xc00bb0ca68 sp=0xc00bb0ca00 pc=0x10c6edf github.com/cockroachdb/pebble/sstable.(*compactionIterator).Next(0xc0057f4528) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1378 +0x5d fp=0xc00bb0caa0 sp=0xc00bb0ca68 pc=0x10c6dfd github.com/cockroachdb/pebble.(*levelIter).Next(0xc02cc3ec80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:806 +0x167 fp=0xc00bb0cac8 sp=0xc00bb0caa0 pc=0x114c207 github.com/cockroachdb/pebble.(*mergingIter).nextEntry(0xc0079dd420, 0xc03b90b680) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:555 +0x6c fp=0xc00bb0cb18 sp=0xc00bb0cac8 pc=0x1150d8c github.com/cockroachdb/pebble.(*mergingIter).Next(0xc0079dd420) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:1107 +0x4e fp=0xc00bb0cb38 sp=0xc00bb0cb18 pc=0x115326e github.com/cockroachdb/pebble.(*compactionIter).iterNext(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:473 github.com/cockroachdb/pebble.(*compactionIter).nextInStripe(0xc014729500) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:496 +0x2e fp=0xc00bb0cb88 sp=0xc00bb0cb38 pc=0x110df4e github.com/cockroachdb/pebble.(*compactionIter).setNext(0xc014729500) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:574 +0x27f fp=0xc00bb0cbe0 sp=0xc00bb0cb88 pc=0x110e43f github.com/cockroachdb/pebble.(*compactionIter).Next(0xc014729500) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:368 +0x705 fp=0xc00bb0cc80 sp=0xc00bb0cbe0 pc=0x110d9a5 github.com/cockroachdb/pebble.(*DB).runCompaction(0xc002a2cd80, 0x132a, 0xc005dda900) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2690 +0x1ea5 fp=0xc00bb0dc60 sp=0xc00bb0cc80 pc=0x1107205 github.com/cockroachdb/pebble.(*DB).compact1(0xc002a2cd80, 0xc005dda900, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2180 +0x1a5 fp=0xc00bb0de80 sp=0xc00bb0dc60 pc=0x1104905 github.com/cockroachdb/pebble.(*DB).compact.func1({0x624a948, 0xc011363f50}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2151 +0xad fp=0xc00bb0dee8 sp=0xc00bb0de80 pc=0x110468d runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc000084440?, 0x6294e00?, 0xc0035c51e0?}}, 0xc00b581f88) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00bb0df58 sp=0xc00bb0dee8 pc=0xfd3223 github.com/cockroachdb/pebble.(*DB).compact(0xc1db00?, 0xc0100cf380?, 0xc00b581fb8?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2148 +0x6b fp=0xc00bb0dfb8 sp=0xc00bb0df58 pc=0x11045ab github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker.func4() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1869 +0x2c fp=0xc00bb0dfe0 sp=0xc00bb0dfb8 pc=0x1102b0c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bb0dfe8 sp=0xc00bb0dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1869 +0x5ea goroutine 480859 [sleep]: runtime.gopark(0x4b8571dc278?, 0x400eb0?, 0xb0?, 0x85?, 0x801c6e7e15?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b09df0 sp=0xc005b09dd0 pc=0x4907d6 time.Sleep(0x1216325bb) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc005b09e30 sp=0xc005b09df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0053a2090, 0x400eb0, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc005b09e88 sp=0xc005b09e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0053a2090, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc005b09ec8 sp=0xc005b09e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc010a9db80?, 0x7, 0xc00184ff70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc005b09fa8 sp=0xc005b09ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc005b09fe0 sp=0xc005b09fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b09fe8 sp=0xc005b09fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 492225 [runnable]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x518?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d308f58 sp=0xc00d308f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f3000d8?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00d308f90 sp=0xc00d308f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f2ffff0, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00d308fb0 sp=0xc00d308f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00fe59180?, 0xc00598b000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00d308fd8 sp=0xc00d308fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00fe59180, {0xc00598b000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00d309058 sp=0xc00d308fd8 pc=0x535efa net.(*netFD).Read(0xc00fe59180, {0xc00598b000?, 0x7b5a68?, 0xc00fe59180?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00d3090a0 sp=0xc00d309058 pc=0x7a20c9 net.(*conn).Read(0xc03234cfb8, {0xc00598b000?, 0x164290f?, 0xc03234cfb8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00d3090e8 sp=0xc00d3090a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0076ed4a0, {0xc00598b000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00d309130 sp=0xc00d3090e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00d439b30, {0xc00598b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00d3091d0 sp=0xc00d309130 pc=0x370f896 bufio.(*Reader).fill(0xc008c88090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00d309208 sp=0xc00d3091d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc008c88090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00d309228 sp=0xc00d309208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc008c88668?, {0x62295c8, 0xc008c88090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00d309260 sp=0xc00d309228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc008c88000, 0xc00654ffc0, 0xc00d309442, 0xc00d309444, {0x623b718, 0xc00d0503f0}, 0xc0059fa000, 0xc00289e7e4, 0xc00d309440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00d309398 sp=0xc00d309260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc008c88000, {0x624a948, 0xc0081a56e0}, 0xc00654ffa0, 0xc0059fa000, 0xc0081a4300, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00d309768 sp=0xc00d309398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0081a56e0}, {0x628a538, 0xc0076ed4a0}, {{{0xc005981022, 0x4}}, 0x1, 0xc0081a4390, 0xc0081a43c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00d3098a8 sp=0xc00d309768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0081a56e0}, {0x628a538?, 0xc0076ed4a0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00d309e20 sp=0xc00d3098a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc001c39ec0?}, {0x628a538, 0xc0076ed4a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00d309eb0 sp=0xc00d309e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc001c39ec0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00d309f30 sp=0xc00d309eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00d309fe0 sp=0xc00d309f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d309fe8 sp=0xc00d309fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 344682 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x106?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00faf8f58 sp=0xc00faf8f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7ea0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00faf8f90 sp=0xc00faf8f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7db8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00faf8fb0 sp=0xc00faf8f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc03f2c7500?, 0xc0059be000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00faf8fd8 sp=0xc00faf8fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc03f2c7500, {0xc0059be000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00faf9058 sp=0xc00faf8fd8 pc=0x535efa net.(*netFD).Read(0xc03f2c7500, {0xc0059be000?, 0x7b5a68?, 0xc03f2c7500?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00faf90a0 sp=0xc00faf9058 pc=0x7a20c9 net.(*conn).Read(0xc00de7d8f0, {0xc0059be000?, 0x164290f?, 0xc00de7d8f0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00faf90e8 sp=0xc00faf90a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01101b200, {0xc0059be000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00faf9130 sp=0xc00faf90e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc007730438, {0xc0059be000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00faf91d0 sp=0xc00faf9130 pc=0x370f896 bufio.(*Reader).fill(0xc003492890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00faf9208 sp=0xc00faf91d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003492890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00faf9228 sp=0xc00faf9208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003492e68?, {0x62295c8, 0xc003492890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00faf9260 sp=0xc00faf9228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003492800, 0xc008e75520, 0xc00faf9442, 0xc00faf9444, {0x623b718, 0xc00d1db950}, 0xc0059fa000, 0xc00f9c1894, 0xc00faf9440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00faf9398 sp=0xc00faf9260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003492800, {0x624a948, 0xc015dc4570}, 0xc008e75500, 0xc0059fa000, 0xc015dc44b0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00faf9768 sp=0xc00faf9398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc015dc4570}, {0x628a538, 0xc01101b200}, {{{0xc0060ca038, 0x4}}, 0x1, 0xc015dc44e0, 0xc015dc4510, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00faf98a8 sp=0xc00faf9768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc015dc4570}, {0x628a538?, 0xc01101b200}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00faf9e20 sp=0xc00faf98a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc015dc43f0?}, {0x628a538, 0xc01101b200}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00faf9eb0 sp=0xc00faf9e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc015dc43f0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00faf9f30 sp=0xc00faf9eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00faf9fe0 sp=0xc00faf9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00faf9fe8 sp=0xc00faf9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 463500 [sleep]: runtime.gopark(0x4b8c48bc082?, 0x801919?, 0x54?, 0xc9?, 0x80b0b255a2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a2c5df0 sp=0xc00a2c5dd0 pc=0x4907d6 time.Sleep(0x14838fe12) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a2c5e30 sp=0xc00a2c5df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc010c413b0, 0x801919, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a2c5e88 sp=0xc00a2c5e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc010c413b0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a2c5ec8 sp=0xc00a2c5e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00dff7f80?, {0xc014fa2000?, 0xc, 0xc00dff7f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a2c5fa8 sp=0xc00a2c5ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a2c5fe0 sp=0xc00a2c5fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a2c5fe8 sp=0xc00a2c5fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498102 [select]: runtime.gopark(0xc00c2d3e60?, 0x3?, 0x0?, 0x0?, 0xc00c2d3e32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c2d3cb8 sp=0xc00c2d3c98 pc=0x4907d6 runtime.selectgo(0xc00c2d3e60, 0xc00c2d3e2c, 0xc012409ad0?, 0x0, 0xc005976e18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c2d3df8 sp=0xc00c2d3cb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc013a4c2d0, {0x624a948, 0xc012409ad0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00c2d3eb0 sp=0xc00c2d3df8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc012409ad0?}) :1 +0x39 fp=0xc00c2d3ed8 sp=0xc00c2d3eb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc012409ad0}, {0x53ddbcd?, 0x6294e00?}, 0xc004203fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00c2d3f50 sp=0xc00c2d3ed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00c2d3fe0 sp=0xc00c2d3f50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c2d3fe8 sp=0xc00c2d3fe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 442766 [sleep]: runtime.gopark(0x4b8cdc58d80?, 0x206747?, 0x8?, 0xb7?, 0x80bc024358?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016109df0 sp=0xc016109dd0 pc=0x4907d6 time.Sleep(0x14a4f1f4a) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc016109e30 sp=0xc016109df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc003ad1fb0, 0x206747, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc016109e88 sp=0xc016109e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc003ad1fb0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc016109ec8 sp=0xc016109e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc0174c7f80?, {0xc00de93900?, 0x20, 0xc0174c7f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc016109fa8 sp=0xc016109ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc016109fe0 sp=0xc016109fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016109fe8 sp=0xc016109fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 424889 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x405?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007dacf58 sp=0xc007dacf38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f3002b8?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc007dacf90 sp=0xc007dacf58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f3001d0, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc007dacfb0 sp=0xc007dacf90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc002fc0680?, 0xc0086d4000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc007dacfd8 sp=0xc007dacfb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc002fc0680, {0xc0086d4000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc007dad058 sp=0xc007dacfd8 pc=0x535efa net.(*netFD).Read(0xc002fc0680, {0xc0086d4000?, 0x7b5a68?, 0xc002fc0680?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc007dad0a0 sp=0xc007dad058 pc=0x7a20c9 net.(*conn).Read(0xc00678eb48, {0xc0086d4000?, 0x164290f?, 0xc00678eb48?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc007dad0e8 sp=0xc007dad0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc010c2b860, {0xc0086d4000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc007dad130 sp=0xc007dad0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0474637a0, {0xc0086d4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc007dad1d0 sp=0xc007dad130 pc=0x370f896 bufio.(*Reader).fill(0xc00a045c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc007dad208 sp=0xc007dad1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00a045c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc007dad228 sp=0xc007dad208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00a046268?, {0x62295c8, 0xc00a045c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc007dad260 sp=0xc007dad228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00a045c00, 0xc00ae1cab0, 0xc007dad442, 0xc007dad444, {0x623b718, 0xc03f0ee990}, 0xc0059fa000, 0xc002242704, 0xc007dad440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc007dad398 sp=0xc007dad260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00a045c00, {0x624a948, 0xc0122c9680}, 0xc00ae1caa0, 0xc0059fa000, 0xc0122c95c0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc007dad768 sp=0xc007dad398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0122c9680}, {0x628a538, 0xc010c2b860}, {{{0xc004434009, 0x4}}, 0x1, 0xc0122c95f0, 0xc0122c9620, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc007dad8a8 sp=0xc007dad768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0122c9680}, {0x628a538?, 0xc010c2b860}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc007dade20 sp=0xc007dad8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0122c9500?}, {0x628a538, 0xc010c2b860}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc007dadeb0 sp=0xc007dade20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0122c9500}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc007dadf30 sp=0xc007dadeb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc007dadfe0 sp=0xc007dadf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007dadfe8 sp=0xc007dadfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498460 [select]: runtime.gopark(0xc01131b208?, 0x2?, 0x0?, 0xa9?, 0xc01131b204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01131b090 sp=0xc01131b070 pc=0x4907d6 runtime.selectgo(0xc01131b208, 0xc01131b200, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01131b1d0 sp=0xc01131b090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc011e31c20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01131b238 sp=0xc01131b1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018ac8f0, {0x51b9a20?, 0xc00d502300}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01131b340 sp=0xc01131b238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x117?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01131b370 sp=0xc01131b340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011e319e0, 0xc01131b448, 0xc01131b418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01131b3c8 sp=0xc01131b370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011e319e0, {0x51b9a20?, 0xc00d502300?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01131b478 sp=0xc01131b3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc011c55650?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0081db950}, {0x51b9a20, 0xc00d502300}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01131b4e0 sp=0xc01131b478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011c55650}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081db950}, {0x51b9a20, 0xc00d502300}, 0x2a286ac68eff604c?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01131b5d0 sp=0xc01131b4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc011c55650?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0081db950?}, {0x51b9a20?, 0xc00d502300?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01131b670 sp=0xc01131b5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc012277358, {0x624a948, 0xc011c55650}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01131b6f0 sp=0xc01131b670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc012277358?}}, {0x624a948, 0xc011c55650}, 0xc0081db950, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01131b7c8 sp=0xc01131b6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xba83830?, {0x624a948?, 0xc011c55650?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00ba83928?}) :1 +0x65 fp=0xc01131b818 sp=0xc01131b7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00df7ad80, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01131bab8 sp=0xc01131b818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00df7ad80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e295af18, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01131bbf0 sp=0xc01131bab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e295af18, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01131c338 sp=0xc01131bbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e295af18, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01131cc38 sp=0xc01131c338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e295af18, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc01131d4a0 sp=0xc01131cc38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e295af18, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01131d8c0 sp=0xc01131d4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc01131d9f8 sp=0xc01131d8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc003210510}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc01131dc00 sp=0xc01131d9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc003210510}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc01131df30 sp=0xc01131dc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01131dfe0 sp=0xc01131df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01131dfe8 sp=0xc01131dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 490513 [runnable]: syscall.Syscall(0x1058d47?, 0xc00829c4d0?, 0x1058da0?, 0xc0057e00c0?) GOROOT/src/syscall/syscall_linux.go:68 +0x27 fp=0xc00bbc8b70 sp=0xc00bbc8b00 pc=0x4d89a7 syscall.Syscall(0x4b, 0x3df, 0x0, 0x0) :1 +0x34 fp=0xc00bbc8ba0 sp=0xc00bbc8b70 pc=0x4d8d74 golang.org/x/sys/unix.Fdatasync(0x1058740?) golang.org/x/sys/unix/external/org_golang_x_sys/unix/zsyscall_linux.go:712 +0x30 fp=0xc00bbc8be8 sp=0xc00bbc8ba0 pc=0xb50910 github.com/cockroachdb/pebble/vfs.(*linuxFile).SyncData(0xc00bbc8c10?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/default_linux.go:59 +0x1d fp=0xc00bbc8c00 sp=0xc00bbc8be8 pc=0x105689d github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).SyncData.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:150 +0x2e fp=0xc00bbc8c20 sp=0xc00bbc8c00 pc=0x1058b0e github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).timeDiskOp(0xc0057e00c0, 0xc00bbc8c88) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:175 +0xd8 fp=0xc00bbc8c68 sp=0xc00bbc8c20 pc=0x1058d38 github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).SyncData(0x101?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:149 +0x4e fp=0xc00bbc8cb0 sp=0xc00bbc8c68 pc=0x1058aae github.com/cockroachdb/pebble/vfs.(*enospcFile).SyncData(0x0?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:409 +0x22 fp=0xc00bbc8cc8 sp=0xc00bbc8cb0 pc=0x1058082 github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl.(*encryptedFile).SyncData(0x4e4837?) :1 +0x2a fp=0xc00bbc8ce0 sp=0xc00bbc8cc8 pc=0x4457e0a github.com/cockroachdb/pebble/vfs.(*syncingFile).Sync(0xc0057e0140?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/syncing_file.go:114 +0x4b fp=0xc00bbc8cf8 sp=0xc00bbc8ce0 pc=0x105f90b github.com/cockroachdb/pebble/record.(*LogWriter).syncWithLatency(0xc005f70000) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:532 +0x43 fp=0xc00bbc8d48 sp=0xc00bbc8cf8 pc=0x10ad783 github.com/cockroachdb/pebble/record.(*LogWriter).flushPending(0xc005f70000, {0xc0054a0a35, 0x101, 0x75d3}, {0xc015c5cb80, 0x0, 0x4efa66?}, 0xb91, 0xb90) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:519 +0x1d0 fp=0xc00bbc8dc0 sp=0xc00bbc8d48 pc=0x10ad550 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc005f70000, {0x4bb35c0, 0x9c5fde0}) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:453 +0x358 fp=0xc00bbc8ef8 sp=0xc00bbc8dc0 pc=0x10ace78 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop-fm({0x624a948?, 0xc0048d40f0?}) :1 +0x39 fp=0xc00bbc8f20 sp=0xc00bbc8ef8 pc=0x10b0099 runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc0000843c0?, 0xc00c483040?, 0xc00296b6c0?}}, 0xc01750ffc0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00bbc8f90 sp=0xc00bbc8f20 pc=0xfd3223 github.com/cockroachdb/pebble/record.NewLogWriter.func2() github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:329 +0x5c fp=0xc00bbc8fe0 sp=0xc00bbc8f90 pc=0x10aca3c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bbc8fe8 sp=0xc00bbc8fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/record.NewLogWriter github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:328 +0x3ca goroutine 427404 [select]: runtime.gopark(0xc00bfb6e00?, 0x2?, 0xd8?, 0xac?, 0xc00bfb6dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bfb6c88 sp=0xc00bfb6c68 pc=0x4907d6 runtime.selectgo(0xc00bfb6e00, 0xc00bfb6df8, 0x10?, 0x0, 0x301?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bfb6dc8 sp=0xc00bfb6c88 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc01ffc3d40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00bfb6e30 sp=0xc00bfb6dc8 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00844d760, {0x51b9a20?, 0xc003165200}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00bfb6f38 sp=0xc00bfb6e30 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2d4?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00bfb6f68 sp=0xc00bfb6f38 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc01ffc37a0, 0xc00bfb7040, 0xc00bfb7010) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00bfb6fc0 sp=0xc00bfb6f68 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc01ffc37a0, {0x51b9a20?, 0xc003165200?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00bfb7070 sp=0xc00bfb6fc0 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00ddf2de0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00dcedc20}, {0x51b9a20, 0xc003165200}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00bfb70d8 sp=0xc00bfb7070 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00ddf2de0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00dcedc20}, {0x51b9a20, 0xc003165200}, 0x3e94efa914a9c7b8?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00bfb71c8 sp=0xc00bfb70d8 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x70d505d36ebb577a?, {0x624a948?, 0xc00ddf2de0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00dcedc20?}, {0x51b9a20?, 0xc003165200?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00bfb7268 sp=0xc00bfb71c8 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0432770c8, {0x624a948, 0xc00ddf2de0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00bfb72e8 sp=0xc00bfb7268 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0432770c8?}}, {0x624a948, 0xc00ddf2de0}, 0xc00dcedc20, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00bfb73c0 sp=0xc00bfb72e8 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xbfb7428?, {0x624a948?, 0xc00ddf2de0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00bfb7520?}) :1 +0x65 fp=0xc00bfb7410 sp=0xc00bfb73c0 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0432c1da0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00bfb76b0 sp=0xc00bfb7410 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0432c1da0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f2a936, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00bfb77e8 sp=0xc00bfb76b0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f2a936, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00bfb7f30 sp=0xc00bfb77e8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f2a936, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00bfb8830 sp=0xc00bfb7f30 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f2a936, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00bfb9098 sp=0xc00bfb8830 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f2a936, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00bfb94b8 sp=0xc00bfb9098 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc006134bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00bfb9710 sp=0xc00bfb94b8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc006134b80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00bfb9838 sp=0xc00bfb9710 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc006134b50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc00bfb9ac8 sp=0xc00bfb9838 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc006134a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc00bfba068 sp=0xc00bfb9ac8 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc006134a50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc00bfba358 sp=0xc00bfba068 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc006134920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc00bfba658 sp=0xc00bfba358 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc006134900?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc00bfba800 sp=0xc00bfba658 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc006134850, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc00bfbaa18 sp=0xc00bfba800 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc006134680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc00bfbac70 sp=0xc00bfbaa18 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00bfbad98 sp=0xc00bfbac70 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00827d760, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc00bfbb060 sp=0xc00bfbad98 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00bfbb170 sp=0xc00bfbb060 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc00ddf2960}, 0xc00bfbb3d8?, 0xc024bdd600) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00bfbb368 sp=0xc00bfbb170 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00827d760, {0x624a948, 0xc00ddf2960}, 0xc03d47c000?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc00bfbb3b0 sp=0xc00bfbb368 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc007393368, {0x624a948, 0xc00ddf2960}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc00bfbb3e8 sp=0xc00bfbb3b0 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*insertNode).BatchedNext(0xc007393340, {{0x624a948, 0xc00ddf2960}, 0xc03d47c000, 0xc00af35e78}) github.com/cockroachdb/cockroach/pkg/sql/insert.go:260 +0x22e fp=0xc00bfbb448 sp=0xc00bfbb3e8 pc=0x341c12e github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc01052a270, {{0x624a948?, 0xc00ddf2960?}, 0xc03d47c000?, 0xc00af35e78?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc00bfbb4a0 sp=0xc00bfbb448 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc00bfbb6a0?, 0x624c938?}, {0x624ceb0, 0xc01052a270}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc00bfbb550 sp=0xc00bfbb4a0 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc00bfbb588 sp=0xc00bfbb550 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc00bfbb6a0, {0x624ceb0?, 0xc01052a270?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc00bfbb618 sp=0xc00bfbb588 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc00bfbb6a0, {0x624ceb0, 0xc01052a270}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc00bfbb668 sp=0xc00bfbb618 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc00ddf2960?}, 0xc03d47c000?, 0xc00af35e78?}, {0x624ceb0, 0xc01052a270}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc00bfbb6e8 sp=0xc00bfbb668 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc0067738c0, {0x624a948, 0xc00ddf2900}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc00bfbb728 sp=0xc00bfbb6e8 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:122 +0x34 fp=0xc00bfbb750 sp=0xc00bfbb728 pc=0x3224ef4 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0074dd7a0?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc00bfbb790 sp=0xc00bfbb750 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc0074dd7a0, {0x624a8a0?, 0xc00dad3680?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:121 +0x70 fp=0xc00bfbb7e8 sp=0xc00bfbb790 pc=0x3224e70 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc0074dd7a0, {0x624a8a0?, 0xc00dad3680?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:728 +0x3a fp=0xc00bfbb828 sp=0xc00bfbb7e8 pc=0x230ec1a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0052e7e00, {0x624a8a0, 0xc00dad3680}, 0xc0074dd7a0?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc00bfbb8e8 sp=0xc00bfbb828 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc00dad3680?}, 0xc00827d760?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc00bfbb988 sp=0xc00bfbb8e8 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc011775c50}, 0xc00dced2c0, 0xc00827d760, 0xc003164c00, 0xc00830c000, 0xc00af361b0, 0xc015333a30) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc00bfbc238 sp=0xc00bfbb988 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x6208b80?, {0x624a948?, 0xc011775c50}, 0x203003?, 0xc00dced2c0, 0x0?, {{0x624ceb0?, 0xc01052a270?}, 0x0?}, 0xc00830c000, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc00bfbc2f8 sp=0xc00bfbc238 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc00af35900?, {0x624a948, 0xc011775c50}, 0xc00af361b0, 0xc00af35e78?, 0xc00af35e78, 0xc00830c000, 0xc00dad35c0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc00bfbc410 sp=0xc00bfbc2f8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00af35900, {0x624a948?, 0xc011775c50}, 0xc00af35e78, 0xc011775c50?, {0x7f275f329680?, 0xc0038612c8}, 0xa7?, 0xc00cc46c28) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc00bfbc518 sp=0xc00bfbc410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00af35900, {0x624a8a0, 0xc00dad2e40}, 0xc00af35e78, {0x7f275f329680, 0xc0038612c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc00bfbc938 sp=0xc00bfbc518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00af35900, {0x624a948?, 0xc00d4c5110?}, {{0x6273140, 0xc0122cd180}, {0xc00a4fb5de, 0x344}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc00bfbd2c8 sp=0xc00bfbc938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc00d4c5110?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc00bfbd380 sp=0xc00bfbd2c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc00d4c5110?}, {0x6273140?, 0xc0122cd180?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc00bfbd4d0 sp=0xc00bfbd380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00af35900, {0x624a948, 0xc00d4c5110}, {{0x6273140, 0xc0122cd180}, {0xc00a4fb5de, 0x344}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc00bfbd648 sp=0xc00bfbd4d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6273140, 0xc0122cd180}, {0xc00a4fb5de, 0x344}, 0x0, 0x1}, {0xc1116e8196babe0b, 0x7feca9ec60, 0x0}, {0xc1116e8196bac832, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc00bfbd708 sp=0xc00bfbd648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00af35900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc00bfbdcb8 sp=0xc00bfbd708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00af35900, {0x624a8a0, 0xc005b3b780}, 0x41153ae000000000?, 0xc003be5dc0?, 0xc003142b70?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00bfbdd70 sp=0xc00bfbdcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc005b3b780?}, {0xc0059fa000?}, 0x3?, 0xc003f3c708?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00bfbde20 sp=0xc00bfbdd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00bfbdfe0 sp=0xc00bfbde20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bfbdfe8 sp=0xc00bfbdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498424 [select]: runtime.gopark(0xc01807fee8?, 0x2?, 0xd0?, 0xa8?, 0xc01807fee4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01807fd70 sp=0xc01807fd50 pc=0x4907d6 runtime.selectgo(0xc01807fee8, 0xc01807fee0, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01807feb0 sp=0xc01807fd70 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc03f542d80) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01807ff18 sp=0xc01807feb0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ec99ce0, {0x51b9a20?, 0xc011795780}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc018080020 sp=0xc01807ff18 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xe6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc018080050 sp=0xc018080020 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc03f542900, 0xc018080128, 0xc0180800f8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0180800a8 sp=0xc018080050 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc03f542900, {0x51b9a20?, 0xc011795780?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc018080158 sp=0xc0180800a8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc006592990?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00ecf4780}, {0x51b9a20, 0xc011795780}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0180801c0 sp=0xc018080158 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc006592990}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00ecf4780}, {0x51b9a20, 0xc011795780}, 0x34eea109993643b?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0180802b0 sp=0xc0180801c0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x17e6661ce229923d?, {0x624a948?, 0xc006592990?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00ecf4780?}, {0x51b9a20?, 0xc011795780?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc018080350 sp=0xc0180802b0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123bf198, {0x624a948, 0xc006592990}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0180803d0 sp=0xc018080350 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123bf198?}}, {0x624a948, 0xc006592990}, 0xc00ecf4780, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0180804a8 sp=0xc0180803d0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xe670510?, {0x624a948?, 0xc006592990?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00e670608?}) :1 +0x65 fp=0xc0180804f8 sp=0xc0180804a8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0153adf80, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc018080798 sp=0xc0180804f8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0153adf80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3973a1d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0180808d0 sp=0xc018080798 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3973a1d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc018081018 sp=0xc0180808d0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3973a1d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc018081918 sp=0xc018081018 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3973a1d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc018082180 sp=0xc018081918 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3973a1d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0180825a0 sp=0xc018082180 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc016883c38, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc0180827f8 sp=0xc0180825a0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc016883c00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc018082920 sp=0xc0180827f8 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc016883bd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc018082bb0 sp=0xc018082920 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc016883ad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc018083150 sp=0xc018082bb0 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc016883ad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc018083440 sp=0xc018083150 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0168839a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc018083740 sp=0xc018083440 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc016883980?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc0180838e8 sp=0xc018083740 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0168838d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc018083b00 sp=0xc0180838e8 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc016883700, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc018083d58 sp=0xc018083b00 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc018083e80 sp=0xc018083d58 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00ec99080, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc018084148 sp=0xc018083e80 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc018084260 sp=0xc018084148 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0077de000, {0x624a948, 0xc006592240}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc018084480 sp=0xc018084260 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0077de000, {0x624a948, 0xc006592240}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc018084778 sp=0xc018084480 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc001de66c0, {0x624a948, 0xc006592240}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc018084858 sp=0xc018084778 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc008e81500, {0x624a948, 0xc006592240}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc018084b30 sp=0xc018084858 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc05492f720) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc018084b88 sp=0xc018084b30 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0089d2700) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc018084ba0 sp=0xc018084b88 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec.spool_false(0xc003928ea0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/sorttopk.eg.go:320 +0x3e fp=0xc018084c40 sp=0xc018084ba0 pc=0x2a12c7e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0x203002?) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/sorttopk.eg.go:73 +0x2c fp=0xc018084c58 sp=0xc018084c40 pc=0x2a11dcc github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc003928ea0) github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:165 +0x65 fp=0xc018084cb8 sp=0xc018084c58 pc=0x29ab645 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecdisk.(*diskSpillerBase).Next.func1() github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecdisk/disk_spiller.go:198 +0x2f fp=0xc018084cd8 sp=0xc018084cb8 pc=0x2c1a54f github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x0?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc018084d18 sp=0xc018084cd8 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecdisk.(*diskSpillerBase).Next(0xc0111a85a0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecdisk/disk_spiller.go:196 +0x68 fp=0xc018084d98 sp=0xc018084d18 pc=0x2c1a3c8 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc010bf0b90) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:135 +0x2e fp=0xc018084de0 sp=0xc018084d98 pc=0x2860a0e github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBytesOp.Next({{{{0x62648e8, 0xc010bf0b90}}, {{0x624a948, 0xc0065921e0}}}, 0xc0109c5b90, 0x7, {0xc0034aa480, 0x10, 0x10}}) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/colexecbase/const.eg.go:213 +0x84 fp=0xc018084ec0 sp=0xc018084de0 pc=0x297eba4 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*constBytesOp).Next(0xc005118f88?) :1 +0x9f fp=0xc018084f60 sp=0xc018084ec0 pc=0x29943bf github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc010bf0c80) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:135 +0x2e fp=0xc018084fa8 sp=0xc018084f60 pc=0x2860a0e github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constInt64Op.Next({{{{0x62648e8, 0xc010bf0c80}}, {{0x624a948, 0xc0065921e0}}}, 0xc0109c5bf0, 0x8, 0x4}) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/colexecbase/const.eg.go:368 +0x74 fp=0xc018085088 sp=0xc018084fa8 pc=0x297f994 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*constInt64Op).Next(0x7f278cfaa5b8?) :1 +0x70 fp=0xc018085108 sp=0xc018085088 pc=0x2995230 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc010bf1400) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:135 +0x2e fp=0xc018085150 sp=0xc018085108 pc=0x2860a0e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*projectInOpBytes).Next(0xc001de67e0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/select_in.eg.go:734 +0x3e fp=0xc018085250 sp=0xc018085150 pc=0x29f3a5e github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc010bf1810) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:135 +0x2e fp=0xc018085298 sp=0xc018085250 pc=0x2860a0e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*projectInOpBytes).Next(0xc001de68a0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/select_in.eg.go:734 +0x3e fp=0xc018085398 sp=0xc018085298 pc=0x29f3a5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc00332c780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc0180853c0 sp=0xc018085398 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc00332c780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc0180853d8 sp=0xc0180853c0 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc0180853f0 sp=0xc0180853d8 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x62841c0?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc018085430 sp=0xc0180853f0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc00332c780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc018085468 sp=0xc018085430 pc=0x29a100c github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc0034a8f00, {{0x624a948?, 0xc006592120?}, 0x0?, 0xc005119538?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc0180854e0 sp=0xc018085468 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00a54d800, {{0x624a948, 0xc006592120}, 0xc01705f000, 0xc00483f178}) github.com/cockroachdb/cockroach/pkg/sql/update.go:164 +0xc4 fp=0xc018085540 sp=0xc0180854e0 pc=0x352d844 github.com/cockroachdb/cockroach/pkg/sql.(*serializeNode).Next(0xc0109c4ff0, {{0x624a948?, 0xc006592120?}, 0xc01705f000?, 0xc00483f178?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:115 +0x5a fp=0xc018085578 sp=0xc018085540 pc=0x3476ffa github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc00b1e18c0) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc018085670 sp=0xc018085578 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).next(0xc003a84ea0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:130 +0x3b fp=0xc0180856a0 sp=0xc018085670 pc=0x3224f5b github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).nextAdapter(0xc003a84ea0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:147 +0x1e fp=0xc0180856b8 sp=0xc0180856a0 pc=0x322501e github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc0180856d0 sp=0xc0180856b8 pc=0x3233486 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x549446b?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc018085710 sp=0xc0180856d0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Next(0xc003a84ea0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:152 +0x3e fp=0xc018085758 sp=0xc018085710 pc=0x32250de github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run({0x624a948, 0xc0065920c0}, {0x626bfc0, 0xc003a84ea0}, {0x6221a60?, 0xc006912a80?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:186 +0x55 fp=0xc0180857a8 sp=0xc018085758 pc=0x2309bf5 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc003a84ea0, {0x624a8a0?, 0xc0089d26c0?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:729 +0x5b fp=0xc0180857e8 sp=0xc0180857a8 pc=0x230ec3b github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0134fa400, {0x624a8a0, 0xc0089d26c0}, 0xc003a84ea0?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc0180858a8 sp=0xc0180857e8 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc0089d26c0?}, 0xc00ec99080?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc018085948 sp=0xc0180858a8 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc0109c50b0}, 0xc00ecf4000, 0xc00ec99080, 0xc011795480, 0xc006912a80, 0xc00483f4b0, 0xc00e2a8c10) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc0180861f8 sp=0xc018085948 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc0109c50b0}, 0xe8?, 0xc00ecf4000, 0xa895921cd46f0701?, {{0x624d0e0?, 0xc0109c4ff0?}, 0x0?}, 0xc006912a80, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0180862b8 sp=0xc0180861f8 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc0109c50b0}, 0xc00483f4b0, 0xc00483f178?, 0xc00483f178, 0xc006912a80, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc0180863d0 sp=0xc0180862b8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00483ec00, {0x624a948?, 0xc0109c50b0}, 0xc00483f178, 0xc0109c50b0?, {0x629ea60?, 0xc008365bc0}, 0xbc?, 0xc00a03e8a8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc0180864d8 sp=0xc0180863d0 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00483ec00, {0x624a8a0, 0xc0089d21c0}, 0xc00483f178, {0x629ea60, 0xc008365bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc0180868f8 sp=0xc0180864d8 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00483ec00, {0x624a948?, 0xc0083654d0?}, {{0x6274800, 0xc011794680}, {0x555828c, 0x103}, 0x3, 0x1}, 0xc007e73e60, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc018087288 sp=0xc0180868f8 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc0083654d0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc018087340 sp=0xc018087288 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc0083654d0?}, {0x6274800?, 0xc011794680?}, 0x624a948?, 0xc0083654d0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc018087490 sp=0xc018087340 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00483ec00, {0x624a948, 0xc0083654d0}, {{0x6274800, 0xc011794680}, {0x555828c, 0x103}, 0x3, 0x1}, 0xc007e73e60, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc018087608 sp=0xc018087490 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00483ec00, {0x624a948, 0xc0083654d0}, {0xc007e73e60, {0xc008365b90, 0x3, 0x3}, {0x0, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc018087728 sp=0xc018087608 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0x0, 0x0}, 0x0, {0xc1116e8196611eeb, 0x7fec504d60, 0x0}, 0x1}, 0xc00483ec00, 0xc018087a98, 0xc018087a88, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc0180878e0 sp=0xc018087728 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00483ec00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc018087e90 sp=0xc0180878e0 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00483ec00, {0x624a948, 0xc008365050}, 0x1054dc0?, 0xc007bc9180?, 0x1054e20?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc018087f48 sp=0xc018087e90 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx.func1() github.com/cockroachdb/cockroach/pkg/sql/internal.go:173 +0xac fp=0xc018087fe0 sp=0xc018087f48 pc=0x342220c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc018087fe8 sp=0xc018087fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx github.com/cockroachdb/cockroach/pkg/sql/internal.go:172 +0x195 goroutine 497357 [sleep]: runtime.gopark(0x4b8ee640cac?, 0xfe2f50?, 0x6?, 0xa?, 0x80e40b6061?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00da2adf0 sp=0xc00da2add0 pc=0x4907d6 time.Sleep(0x151b9c145) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00da2ae30 sp=0xc00da2adf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00c840030, 0xfe2f50, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00da2ae88 sp=0xc00da2ae30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00c840030, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00da2aec8 sp=0xc00da2ae88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc015d70500?, 0x4, 0xc00a351f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00da2afa8 sp=0xc00da2aec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00da2afe0 sp=0xc00da2afa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00da2afe8 sp=0xc00da2afe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 424890 [select]: runtime.gopark(0xc016074e00?, 0x2?, 0xc0?, 0xb2?, 0xc016074dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016074c88 sp=0xc016074c68 pc=0x4907d6 runtime.selectgo(0xc016074e00, 0xc016074df8, 0x10?, 0x0, 0x1001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc016074dc8 sp=0xc016074c88 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00cce78c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc016074e30 sp=0xc016074dc8 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ef83e40, {0x51b9a20?, 0xc007f9f900}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc016074f38 sp=0xc016074e30 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xde2?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc016074f68 sp=0xc016074f38 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc00cce7440, 0xc016075040, 0xc016075010) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc016074fc0 sp=0xc016074f68 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00cce7440, {0x51b9a20?, 0xc007f9f900?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc016075070 sp=0xc016074fc0 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc0015dfd10?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0106163c0}, {0x51b9a20, 0xc007f9f900}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0160750d8 sp=0xc016075070 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0015dfd10}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0106163c0}, {0x51b9a20, 0xc007f9f900}, 0x41314b4cc49ecaa0?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0160751c8 sp=0xc0160750d8 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0xde9e5f6a5ea1bcc?, {0x624a948?, 0xc0015dfd10?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0106163c0?}, {0x51b9a20?, 0xc007f9f900?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc016075268 sp=0xc0160751c8 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc046762158, {0x624a948, 0xc0015dfd10}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0160752e8 sp=0xc016075268 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc046762158?}}, {0x624a948, 0xc0015dfd10}, 0xc0106163c0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0160753c0 sp=0xc0160752e8 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x16075428?, {0x624a948?, 0xc0015dfd10?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc016075520?}) :1 +0x65 fp=0xc016075410 sp=0xc0160753c0 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0061b86c0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0160756b0 sp=0xc016075410 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0061b86c0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2cdd97f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0160757e8 sp=0xc0160756b0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2cdd97f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc016075f30 sp=0xc0160757e8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2cdd97f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc016076830 sp=0xc016075f30 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2cdd97f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc016077098 sp=0xc016076830 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2cdd97f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0160774b8 sp=0xc016077098 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0069920b8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc016077710 sp=0xc0160774b8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc006992080, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc016077838 sp=0xc016077710 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc006992050, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc016077ac8 sp=0xc016077838 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc006991f50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc016078068 sp=0xc016077ac8 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc006991f50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc016078358 sp=0xc016078068 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc006991e20, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc016078658 sp=0xc016078358 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc006991e00?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc016078800 sp=0xc016078658 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc006991d50, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc016078a18 sp=0xc016078800 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc006991b80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc016078c70 sp=0xc016078a18 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc016078d98 sp=0xc016078c70 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc012103ce0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc016079060 sp=0xc016078d98 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc016079170 sp=0xc016079060 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc002fd2270}, 0x456936?, 0xc009feb600) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc016079368 sp=0xc016079170 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc012103ce0, {0x624a948, 0xc002fd2270}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc0160793b0 sp=0xc016079368 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc005298e28, {0x624a948, 0xc002fd2270}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc0160793e8 sp=0xc0160793b0 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc005298e00, {{0x624a948, 0xc002fd2270}, 0xc007b30400, 0xc00361be78}) github.com/cockroachdb/cockroach/pkg/sql/update.go:197 +0x225 fp=0xc016079448 sp=0xc0160793e8 pc=0x352d9a5 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc047462150, {{0x624a948?, 0xc002fd2270?}, 0xc007b30400?, 0xc00361be78?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc0160794a0 sp=0xc016079448 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc0160796a0?, 0x624d578?}, {0x624ceb0, 0xc047462150}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc016079550 sp=0xc0160794a0 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc016079588 sp=0xc016079550 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0160796a0, {0x624ceb0?, 0xc047462150?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc016079618 sp=0xc016079588 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0160796a0, {0x624ceb0, 0xc047462150}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc016079668 sp=0xc016079618 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc002fd2270?}, 0xc007b30400?, 0xc00361be78?}, {0x624ceb0, 0xc047462150}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc0160796e8 sp=0xc016079668 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00f30b080, {0x624a948, 0xc002fd2240}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc016079728 sp=0xc0160796e8 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:122 +0x34 fp=0xc016079750 sp=0xc016079728 pc=0x3224ef4 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0044c0000?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc016079790 sp=0xc016079750 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc0044c0000, {0x624a8a0?, 0xc01388f740?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:121 +0x70 fp=0xc0160797e8 sp=0xc016079790 pc=0x3224e70 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc0044c0000, {0x624a8a0?, 0xc01388f740?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:728 +0x3a fp=0xc016079828 sp=0xc0160797e8 pc=0x230ec1a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00401cc00, {0x624a8a0, 0xc01388f740}, 0xc0044c0000?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc0160798e8 sp=0xc016079828 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc01388f740?}, 0xc012103ce0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc016079988 sp=0xc0160798e8 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc0238072c0}, 0xc004bb60f0, 0xc012103ce0, 0xc00ec16800, 0xc00a2e2e00, 0xc00361c1b0, 0xc0468f8840) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc01607a238 sp=0xc016079988 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc0238072c0}, 0xe8?, 0xc004bb60f0, 0x5256778d63f92501?, {{0x624ceb0?, 0xc047462150?}, 0x0?}, 0xc00a2e2e00, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc01607a2f8 sp=0xc01607a238 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc0238072c0}, 0xc00361c1b0, 0xc00361be78?, 0xc00361be78, 0xc00a2e2e00, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc01607a410 sp=0xc01607a2f8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00361b900, {0x624a948?, 0xc0238072c0}, 0xc00361be78, 0xc0238072c0?, {0x7f275f329680?, 0xc00a0462c8}, 0xaa?, 0xc042b93478) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc01607a518 sp=0xc01607a410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00361b900, {0x624a8a0, 0xc01388f200}, 0xc00361be78, {0x7f275f329680, 0xc00a0462c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc01607a938 sp=0xc01607a518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00361b900, {0x624a948?, 0xc0383fd350?}, {{0x6274800, 0xc00ec16100}, {0xc002cda9c0, 0xe2}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc01607b2c8 sp=0xc01607a938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc0383fd350?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc01607b380 sp=0xc01607b2c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc0383fd350?}, {0x6274800?, 0xc00ec16100?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc01607b4d0 sp=0xc01607b380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00361b900, {0x624a948, 0xc0383fd350}, {{0x6274800, 0xc00ec16100}, {0xc002cda9c0, 0xe2}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc01607b648 sp=0xc01607b4d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6274800, 0xc00ec16100}, {0xc002cda9c0, 0xe2}, 0x0, 0x1}, {0xc1116e81923925e4, 0x7fe8285447, 0x0}, {0xc1116e8192392d86, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc01607b708 sp=0xc01607b648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00361b900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc01607bcb8 sp=0xc01607b708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00361b900, {0x624a8a0, 0xc0051a1700}, 0x411481d800000000?, 0xc004a8fdc0?, 0xc00ae1cb90?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc01607bd70 sp=0xc01607bcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0051a1700?}, {0xc0059fa000?}, 0x3?, 0xc0072d02e8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc01607be20 sp=0xc01607bd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc01607bfe0 sp=0xc01607be20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01607bfe8 sp=0xc01607bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 4831 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0xb64?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fcd2f58 sp=0xc00fcd2f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f243880?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00fcd2f90 sp=0xc00fcd2f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f243798, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00fcd2fb0 sp=0xc00fcd2f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00458db80?, 0xc0034fe000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00fcd2fd8 sp=0xc00fcd2fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00458db80, {0xc0034fe000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00fcd3058 sp=0xc00fcd2fd8 pc=0x535efa net.(*netFD).Read(0xc00458db80, {0xc0034fe000?, 0x7b5a68?, 0xc00458db80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00fcd30a0 sp=0xc00fcd3058 pc=0x7a20c9 net.(*conn).Read(0xc012085ae8, {0xc0034fe000?, 0x164290f?, 0xc012085ae8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00fcd30e8 sp=0xc00fcd30a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0077af440, {0xc0034fe000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00fcd3130 sp=0xc00fcd30e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0075cac60, {0xc0034fe000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00fcd31d0 sp=0xc00fcd3130 pc=0x370f896 bufio.(*Reader).fill(0xc0014d1490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00fcd3208 sp=0xc00fcd31d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0014d1490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00fcd3228 sp=0xc00fcd3208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0014d1a68?, {0x62295c8, 0xc0014d1490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00fcd3260 sp=0xc00fcd3228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0014d1400, 0xc00090b0c0, 0xc00fcd3442, 0xc00fcd3444, {0x623b718, 0xc005eb6000}, 0xc0059fa000, 0xc006bdaf50, 0xc00fcd3440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00fcd3398 sp=0xc00fcd3260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0014d1400, {0x624a948, 0xc0058677a0}, 0xc00090b0b0, 0xc0059fa000, 0xc005867710, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00fcd3768 sp=0xc00fcd3398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0058677a0}, {0x628a538, 0xc0077af440}, {{{0xc0034e4009, 0x4}}, 0x1, 0xc005867740, 0xc005867770, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00fcd38a8 sp=0xc00fcd3768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0058677a0}, {0x628a538?, 0xc0077af440}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00fcd3e20 sp=0xc00fcd38a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc005867650?}, {0x628a538, 0xc0077af440}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00fcd3eb0 sp=0xc00fcd3e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc005867650}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00fcd3f30 sp=0xc00fcd3eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00fcd3fe0 sp=0xc00fcd3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fcd3fe8 sp=0xc00fcd3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 3985 [select]: runtime.gopark(0xc001832a08?, 0x2?, 0x88?, 0xf2?, 0xc0018329ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001832870 sp=0xc001832850 pc=0x4907d6 runtime.selectgo(0xc001832a08, 0xc0018329e8, 0x16e9e6c?, 0x0, 0xc010ff2417?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0018329b0 sp=0xc001832870 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc004d5c910, {0xc001664640, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc001832a38 sp=0xc0018329b0 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc004d5c910, {0xc001664640?, 0xc000ab72c0?, 0xc001832a80?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc001832a78 sp=0xc001832a38 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0016638f0, {0xc001664640?, 0xc001832af8?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc001832ac0 sp=0xc001832a78 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc0016638f0}, {0xc001664640, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc001832b08 sp=0xc001832ac0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0050845a0, {0xc001664640, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc001832b48 sp=0xc001832b08 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc001664630, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc001832bc8 sp=0xc001832b48 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x4908c0?, 0xc0050845a0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc001832c60 sp=0xc001832bc8 pc=0xc19546 google.golang.org/grpc.recv(0xc001832d10?, {0x7f275f301038, 0xc0000af170}, 0x0?, {0x0?, 0x0?}, {0x512da60, 0xc008356600}, 0x0?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc001832cd8 sp=0xc001832c60 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc003d528f0, {0x512da60?, 0xc008356600}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc001832e08 sp=0xc001832cd8 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x4618e7?, {0x512da60?, 0xc008356600?}) :1 +0x34 fp=0xc001832e30 sp=0xc001832e08 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc003450d30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:263 +0x4c fp=0xc001832e60 sp=0xc001832e30 pc=0x1d4b66c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:91 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x628a958?, 0xc003450d30}, 0xc000c62700, {0x624a948, 0xc001663e60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:289 +0xa6 fp=0xc001832ee0 sp=0xc001832e60 pc=0x1d82d66 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x624a948?, 0xc001663e60?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:308 +0x45 fp=0xc001832f30 sp=0xc001832ee0 pc=0x1d82c65 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001832fe0 sp=0xc001832f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001832fe8 sp=0xc001832fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 87448 [semacquire, 6 minutes]: runtime.gopark(0xc00b2f7a68?, 0x46159f?, 0x0?, 0xd8?, 0xc01096ef06?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cf599f0 sp=0xc00cf599d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc00c7c4850, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00cf59a58 sp=0xc00cf599f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00cf59a88 sp=0xc00cf59a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc00b2f7ae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00cf59ab0 sp=0xc00cf59a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc00c7c4840) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc00cf59ac8 sp=0xc00cf59ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00c7c4840?, {0x624a8a0?, 0xc00c7c4800?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc00cf59af8 sp=0xc00cf59ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc003ce5050?}, {0xc00b2f7c68, 0x2, 0xc003ce5050?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc00cf59b48 sp=0xc00cf59af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc006663600, {0x624a948, 0xc003ce5050}, 0xc00a683e80) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc00cf59e68 sp=0xc00cf59b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc010d81e00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc00cf59f30 sp=0xc00cf59e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cf59fe0 sp=0xc00cf59f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cf59fe8 sp=0xc00cf59fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 385961 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00b2f1640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b2f15d0 sp=0xc00b2f15b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc000a0e650, 0x32e) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00b2f1618 sp=0xc00b2f15d0 pc=0x4be30c sync.(*Cond).Wait(0xc0077e9000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00b2f1650 sp=0xc00b2f1618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0077e9668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc00b2f1708 sp=0xc00b2f1650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0038d7300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc00b2f1cb8 sp=0xc00b2f1708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0038d7300, {0x624a8a0, 0xc0033c0080}, 0x4118be2c00000000?, 0xc00b4bfdc0?, 0xc047499110?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00b2f1d70 sp=0xc00b2f1cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0033c0080?}, {0xc0059fa000?}, 0x3?, 0xc0068c3500?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00b2f1e20 sp=0xc00b2f1d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00b2f1fe0 sp=0xc00b2f1e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b2f1fe8 sp=0xc00b2f1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 49109 [select]: runtime.gopark(0xc0079a3f70?, 0x4?, 0x4?, 0x0?, 0xc0079a3ed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0079a3d18 sp=0xc0079a3cf8 pc=0x4907d6 runtime.selectgo(0xc0079a3f70, 0xc0079a3ec8, 0x6242576e45706a6c?, 0x0, 0x78616b5369447a66?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0079a3e58 sp=0xc0079a3d18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0074271e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc0079a3fc8 sp=0xc0079a3e58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc0079a3fe0 sp=0xc0079a3fc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0079a3fe8 sp=0xc0079a3fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 120462 [select]: runtime.gopark(0xc001368f70?, 0x4?, 0x6?, 0x0?, 0xc001368ed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001368d18 sp=0xc001368cf8 pc=0x4907d6 runtime.selectgo(0xc001368f70, 0xc001368ec8, 0xc000acee70?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001368e58 sp=0xc001368d18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01479a4e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc001368fc8 sp=0xc001368e58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc001368fe0 sp=0xc001368fc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001368fe8 sp=0xc001368fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 498397 [select]: runtime.gopark(0xc00a9262d8?, 0x2?, 0x40?, 0xc5?, 0xc00a9262d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a926160 sp=0xc00a926140 pc=0x4907d6 runtime.selectgo(0xc00a9262d8, 0xc00a9262d0, 0x10?, 0x0, 0x301?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a9262a0 sp=0xc00a926160 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc030e6a120) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00a926308 sp=0xc00a9262a0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0038746e0, {0x51b9a20?, 0xc0178af200}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00a926410 sp=0xc00a926308 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x345?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00a926440 sp=0xc00a926410 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc017ca7e60, 0xc00a926518, 0xc00a9264e8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00a926498 sp=0xc00a926440 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc017ca7e60, {0x51b9a20?, 0xc0178af200?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00a926548 sp=0xc00a926498 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00314a8a0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0134f0d20}, {0x51b9a20, 0xc0178af200}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00a9265b0 sp=0xc00a926548 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00314a8a0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0134f0d20}, {0x51b9a20, 0xc0178af200}, 0x6ffd51e80b087e00?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00a9266a0 sp=0xc00a9265b0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x4f4b56d8fd62aae4?, {0x624a948?, 0xc00314a8a0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0134f0d20?}, {0x51b9a20?, 0xc0178af200?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00a926740 sp=0xc00a9266a0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123be980, {0x624a948, 0xc00314a8a0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00a9267c0 sp=0xc00a926740 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123be980?}}, {0x624a948, 0xc00314a8a0}, 0xc0134f0d20, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00a926898 sp=0xc00a9267c0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xa444900?, {0x624a948?, 0xc00314a8a0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00a4449f8?}) :1 +0x65 fp=0xc00a9268e8 sp=0xc00a926898 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00f74c780, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00a926b88 sp=0xc00a9268e8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00f74c780, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0af9890, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00a926cc0 sp=0xc00a926b88 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0af9890, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00a927408 sp=0xc00a926cc0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0af9890, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00a927d08 sp=0xc00a927408 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x624a948, 0xc00314a8a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1506 +0x1bc fp=0xc00a927f30 sp=0xc00a927d08 pc=0x19ca1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a927fe0 sp=0xc00a927f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a927fe8 sp=0xc00a927fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 488578 [sleep]: runtime.gopark(0x4b94472947e?, 0x200561e?, 0x65?, 0xb1?, 0x812da5d1b8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b60fdf0 sp=0xc00b60fdd0 pc=0x4907d6 time.Sleep(0x14545a84e) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00b60fe30 sp=0xc00b60fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc006dd69c0, 0x200561e, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00b60fe88 sp=0xc00b60fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc006dd69c0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00b60fec8 sp=0xc00b60fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00c67f400?, 0xa, 0xc00143cfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00b60ffa8 sp=0xc00b60fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00b60ffe0 sp=0xc00b60ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b60ffe8 sp=0xc00b60ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 4013 [select]: runtime.gopark(0xc011529db8?, 0x3?, 0x64?, 0x0?, 0xc011529d32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00feb7b90 sp=0xc00feb7b70 pc=0x4907d6 runtime.selectgo(0xc00feb7db8, 0xc011529d2c, 0xc0048ba9f0?, 0x0, 0x7bfe7d?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00feb7cd0 sp=0xc00feb7b90 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000288fc0, {0x624a948, 0xc0048ba9f0}, 0xc0076a2eb0, {0xc0068e4288, 0x12}, 0xc0035b9ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1971 +0x366 fp=0xc00feb7e68 sp=0xc00feb7cd0 pc=0x16613e6 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x624a948, 0xc0048ba9f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1879 +0x8d fp=0xc00feb7f30 sp=0xc00feb7e68 pc=0x1660e4d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00feb7fe0 sp=0xc00feb7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00feb7fe8 sp=0xc00feb7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 4081 [select]: runtime.gopark(0xc002d92f48?, 0x2?, 0x6?, 0x0?, 0xc002d92ef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002d92d50 sp=0xc002d92d30 pc=0x4907d6 runtime.selectgo(0xc002d92f48, 0xc002d92ef0, 0x1df6940?, 0x0, 0xc0050bd740?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002d92e90 sp=0xc002d92d50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0045c65a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc002d92fc8 sp=0xc002d92e90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc002d92fe0 sp=0xc002d92fc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002d92fe8 sp=0xc002d92fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 484520 [sleep]: runtime.gopark(0x4b8ccf2244b?, 0x5a12ee?, 0x59?, 0x4a?, 0x80bb10d6b6?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006713df0 sp=0xc006713dd0 pc=0x4907d6 time.Sleep(0x14a311bef) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc006713e30 sp=0xc006713df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc010e10270, 0x5a12ee, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc006713e88 sp=0xc006713e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc010e10270, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc006713ec8 sp=0xc006713e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc0308bf180?, 0x6, 0xc00c246fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc006713fa8 sp=0xc006713ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc006713fe0 sp=0xc006713fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006713fe8 sp=0xc006713fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 488149 [sleep]: runtime.gopark(0x4b9152f749a?, 0x8a1bb8?, 0x96?, 0xa9?, 0x80fac5fff3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b771df0 sp=0xc00b771dd0 pc=0x4907d6 time.Sleep(0x141a8f3e7) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00b771e30 sp=0xc00b771df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc013d46e40, 0x8a1bb8, {0x4618e7?, 0x203004?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00b771e88 sp=0xc00b771e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc013d46e40, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00b771ec8 sp=0xc00b771e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc011afa780?, 0xb, 0xc0000947b8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00b771fa8 sp=0xc00b771ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00b771fe0 sp=0xc00b771fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b771fe8 sp=0xc00b771fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498485 [runnable]: runtime.gopark(0xae1b6401960?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b0d5120 sp=0xc00b0d5100 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x5?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00b0d5188 sp=0xc00b0d5120 pc=0x4a17de sync.runtime_SemacquireMutex(0xf?, 0x60?, 0x17?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00b0d51b8 sp=0xc00b0d5188 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00b0d5210 sp=0xc00b0d51b8 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x10?, 0xc0159a6680?, 0xa13c2?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00b0d5258 sp=0xc00b0d5210 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00858db80, {0x8?, 0x11?}, 0x0, 0x0, 0xc00d99c810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00b0d5358 sp=0xc00b0d5258 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2727 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0068ac000, {0xc003d78c00, 0x6, 0x40}, {0xc0159a6680, 0x6, 0x1f}, 0x60?, 0x61?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:847 +0xf8 fp=0xc00b0d53c0 sp=0xc00b0d5358 pc=0x10c5078 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc00d99d940?, {0xc003d78c00?, 0x1?, 0x1f?}, {0xc0159a6680?, 0x10?, 0x1f?}, 0x0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00b0d5410 sp=0xc00b0d53c0 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00d99d940, {0xc003d78c00, 0x6, 0x40}, {0xc0159a6680, 0x6, 0x1f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00b0d5460 sp=0xc00b0d5410 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00d99ca20, {0xc0159a6680?, 0xc00574e028?, 0x0?}, 0x5?, 0x5?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00b0d5590 sp=0xc00b0d5460 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00d99ca20, {0xc003d78c00?, 0x0?, 0x6?}, {0xc0159a6680?, 0x0?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00b0d55d0 sp=0xc00b0d5590 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc00d99c888, {0xc003d78c00?, 0x40?, 0xc0159a6680?}, {0xc0159a6680, 0x6, 0x1f}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00b0d5628 sp=0xc00b0d55d0 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00d99c500, {0xc0159a6680, 0x6, 0x1f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00b0d56c0 sp=0xc00b0d5628 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc00574e028, {{0xc015fb4880, 0x5, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00b0d5710 sp=0xc00b0d56c0 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekIntentGE(0xc00b99d800, {0xc015fb4880, 0x5, 0x8}, {0x6d, 0x2, 0xdd, 0xd, 0x53, 0x8f, ...}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:540 +0x117 fp=0xc00b0d5838 sp=0xc00b0d5710 pc=0x180d357 github.com/cockroachdb/cockroach/pkg/storage.MVCCResolveWriteIntent({0x624a948, 0xc00564d1a0}, {0x7f275f2b7158?, 0xc00574e000?}, 0x0?, {{{0xc015fb4880, 0x5, 0x8}, {0x0, 0x0, ...}}, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3968 +0x12b fp=0xc00b0d5990 sp=0xc00b0d5838 pc=0x182dfab github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.resolveLocalLocks.func1(0xc00b0d5ef8, 0xc00b0d5f48, 0xc00b0d5f80, 0x94?, 0xd4?, {0x624a948, 0xc00564d1a0}, {0x7f275f2b7158, 0xc00574e000}, 0xc00c684820, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:557 +0x405 fp=0xc00b0d5e58 sp=0xc00b0d5990 pc=0x1ce98e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.resolveLocalLocks({0x624a948, 0xc00564d1a0}, 0xc00811bd50, {0x7f275f2b7158, 0xc00574e000}, 0x0?, 0xc000073680, 0xc0174fd200, {0x62cc4b0, 0xc016717a80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:610 +0x34f fp=0xc00b0d6050 sp=0xc00b0d5e58 pc=0x1ce92cf github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.EndTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc016717a80}, {{0x17600a76a815945c, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:412 +0xf6a fp=0xc00b0d76d0 sp=0xc00b0d6050 pc=0x1ce868a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76a815945c, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc00b0d7f60 sp=0xc00b0d76d0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc017226b40, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00b0d9dd0 sp=0xc00b0d7f60 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00dedbae0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00b0da420 sp=0xc00b0d9dd0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00dedbae0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00b0da8f8 sp=0xc00b0da420 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76c5e2f95c, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00b0db778 sp=0xc00b0da8f8 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc002100400?, {0x624a948, 0xc00564d1a0}, {0xc010cb8c98, 0x8}, 0xc017226b40, 0x8?, 0x0?, {{0x17600a76c5e2f95c, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00b0dc3f8 sp=0xc00b0db778 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc00564d1a0}, {0xc010cb8c98, 0x8}, 0xc017226b40, 0x400000004?, 0xc00b0dd448, {{0x17600a76c5e2f95c, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00b0dc4b0 sp=0xc00b0dc3f8 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc003861800, {0x624a948, 0xc00564d1a0}, 0xc017226b40, 0xc00dedbae0, 0xc00b0dd448, {{0x17600a76c5e2f95c, 0x0, 0x0}, {0x17600a76a815945c, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00b0dce78 sp=0xc00b0dc4b0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc00564d1a0}, 0xc017226b40, 0xc00dedbae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00b0dd5e8 sp=0xc00b0dce78 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00564d1a0}, 0xc017226b40, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00b0dd908 sp=0xc00b0dd5e8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76a815945c, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00b0ddcd0 sp=0xc00b0dd908 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76a815945c, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00b0de658 sp=0xc00b0ddcd0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc006f10590?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00b0de8a0 sp=0xc00b0de658 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00564d140}, {0x4b29600?}, 0xc017226a50) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00b0def10 sp=0xc00b0de8a0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00564d0b0}, 0xc017226a50) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00b0defb8 sp=0xc00b0def10 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00564d0b0?}, {0x52f0080?, 0xc017226a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00b0defe8 sp=0xc00b0defb8 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00564d0b0}, {0x52f0080, 0xc017226a50}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00b0df318 sp=0xc00b0defe8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d0b0?}, {0x52f0080?, 0xc017226a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00b0df358 sp=0xc00b0df318 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00564d0b0}, {0x52f0080, 0xc017226a50}, 0xc00c88d3f0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00b0df388 sp=0xc00b0df358 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d0b0?}, {0x52f0080?, 0xc017226a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00b0df3c8 sp=0xc00b0df388 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00564d0b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00b0df400 sp=0xc00b0df3c8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00564d0b0}, {0xc00601a000?, 0x53d9bbc?}, 0xc00c88d4b8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00b0df478 sp=0xc00b0df400 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00564d0b0?}, {0x52f0080?, 0xc017226a50?}, 0x2?, 0x52f0080?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00b0df4f0 sp=0xc00b0df478 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00564d0b0?}, {0x52f0080?, 0xc017226a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00b0df530 sp=0xc00b0df4f0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00564d0b0?}, {0xc00564d0b0?, 0x4?}, {0x52f0080?, 0xc017226a50?}, {0x51b9a20?, 0xc000611000}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00b0df570 sp=0xc00b0df530 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00564d0b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017226a50}, {0x51b9a20, 0xc000611000}, 0xc00c88d6e8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00b0df660 sp=0xc00b0df570 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00564d0b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017226a50}, {0x51b9a20, 0xc000611000}, 0xc00c88d7c8?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00b0df6f8 sp=0xc00b0df660 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00564d050}, 0xc017226a50, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00b0df778 sp=0xc00b0df6f8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x0?, {0x624a948?, 0xc00564d050?}, 0xc88d830?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00b0df800 sp=0xc00b0df778 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0078b9e60, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00b0dfaa0 sp=0xc00b0df800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0078b9e60, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00b0dfbd8 sp=0xc00b0dfaa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00b0e0320 sp=0xc00b0dfbd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00b0e0c20 sp=0xc00b0e0320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00b0e1488 sp=0xc00b0e0c20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e456baf7, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00b0e18a8 sp=0xc00b0e1488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00a861138, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00b0e1b00 sp=0xc00b0e18a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00a861100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00b0e1c28 sp=0xc00b0e1b00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc00564ccc0}, {0x7f275f30db68, 0xc00a861100}, 0xc003a2fb00, {0xc017226960, 0x4, 0x5}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc00b0e1e60 sp=0xc00b0e1c28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc00564ccc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc00b0e1f30 sp=0xc00b0e1e60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b0e1fe0 sp=0xc00b0e1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b0e1fe8 sp=0xc00b0e1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115788 [select]: runtime.gopark(0xc008d78d68?, 0x2?, 0x5?, 0x0?, 0xc008d78d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008d78bb8 sp=0xc008d78b98 pc=0x4907d6 runtime.selectgo(0xc008d78d68, 0xc008d78d30, 0x8?, 0x0, 0xc0125ae000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008d78cf8 sp=0xc008d78bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc007323450, {0xc015b2e820, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc008d78d98 sp=0xc008d78cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc007323450, {0xc015b2e820?, 0xc005939db8?, 0xc008d78de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc008d78dd8 sp=0xc008d78d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0071f79e0, {0xc015b2e820?, 0xc008d78e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc008d78e20 sp=0xc008d78dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc0071f79e0}, {0xc015b2e820, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc008d78e68 sp=0xc008d78e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0115cd680, {0xc015b2e820, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc008d78ea8 sp=0xc008d78e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc015b2e810, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc008d78f28 sp=0xc008d78ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc01017b028?, 0xc0115cd680, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc008d78fc0 sp=0xc008d78f28 pc=0xc19546 google.golang.org/grpc.recv(0x6?, {0x7f275f301038, 0xc0000af170}, 0xc2b369?, {0x0?, 0x0?}, {0x51d3160, 0xc025aa9140}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc008d79038 sp=0xc008d78fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc002db48f0, {0x51d3160?, 0xc025aa9140}, 0x5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc008d79140 sp=0xc008d79038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc008d79170 sp=0xc008d79140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0115cd440, 0xc008d79248, 0xc008d79218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc008d791c8 sp=0xc008d79170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0115cd440, {0x51d3160?, 0xc025aa9140?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc008d79278 sp=0xc008d791c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc011929340, {0x51d3160?, 0xc025aa9140?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc008d792c0 sp=0xc008d79278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc014294c40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc008d792f0 sp=0xc008d792c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc008d79310 sp=0xc008d792f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0073232c0, 0xc008d79780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc008d793b8 sp=0xc008d79310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc00c7c5280?}, {{0xc004c213e8, 0x1, 0x8}, {0xc004c213f0, 0x1, 0x8}}, {0x17600a17c78406fa, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc008d799a0 sp=0xc008d793b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc00c7c5280}, 0xc014351b30, 0xc004c213e8?, {{0xc004c213e8, 0x1, 0x8}, {0xc004c213f0, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc008d79e80 sp=0xc008d799a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc00c7c5280?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc008d79f58 sp=0xc008d79e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc008d79f78 sp=0xc008d79f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc008d79fe0 sp=0xc008d79f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008d79fe8 sp=0xc008d79fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 11656 [select]: runtime.gopark(0xc00fcba7a0?, 0x2?, 0x40?, 0xae?, 0xc00fcba79c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fcba628 sp=0xc00fcba608 pc=0x4907d6 runtime.selectgo(0xc00fcba7a0, 0xc00fcba798, 0x10?, 0x0, 0x24001?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fcba768 sp=0xc00fcba628 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc013150360) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00fcba7d0 sp=0xc00fcba768 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018acbb0, {0x51b9a20?, 0xc00d502800}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00fcba8d8 sp=0xc00fcba7d0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x20136?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00fcba908 sp=0xc00fcba8d8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc013150000, 0xc00fcba9e0, 0xc00fcba9b0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00fcba960 sp=0xc00fcba908 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc013150000, {0x51b9a20?, 0xc00d502800?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00fcbaa10 sp=0xc00fcba960 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03064aab0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0120d21e0}, {0x51b9a20, 0xc00d502800}, 0x0?, {0xc00a393c00, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00fcbaa78 sp=0xc00fcbaa10 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03064aab0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0120d21e0}, {0x51b9a20, 0xc00d502800}, 0x1ccf256d3a0242fd?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00fcbab68 sp=0xc00fcbaa78 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x6c9a145413083e43?, {0x624a948?, 0xc03064aab0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0120d21e0?}, {0x51b9a20?, 0xc00d502800?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00fcbac08 sp=0xc00fcbab68 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0122773c8, {0x624a948, 0xc03064aab0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00fcbac88 sp=0xc00fcbac08 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0122773c8?}}, {0x624a948, 0xc03064aab0}, 0xc0120d21e0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00fcbad60 sp=0xc00fcbac88 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xfcbadc8?, {0x624a948?, 0xc03064aab0?}, 0x624a948?, {0x0?, 0xc005751d20?, 0xc00fcbaec0?}) :1 +0x65 fp=0xc00fcbadb0 sp=0xc00fcbad60 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc004fd6600, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00fcbb050 sp=0xc00fcbadb0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc004fd6600, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00fcbb188 sp=0xc00fcbb050 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00fcbb8d0 sp=0xc00fcbb188 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00fcbc1d0 sp=0xc00fcbb8d0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1403 +0x1393 fp=0xc00fcbca38 sp=0xc00fcbc1d0 pc=0x19c8fb3 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:947 +0xaaf fp=0xc00fcbce10 sp=0xc00fcbca38 pc=0x19c736f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3c9520d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:843 +0x585 fp=0xc00fcbd230 sp=0xc00fcbce10 pc=0x19c5f45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0042a56b8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00fcbd488 sp=0xc00fcbd230 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0042a5680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00fcbd5b0 sp=0xc00fcbd488 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0042a5650, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x36e fp=0xc00fcbd840 sp=0xc00fcbd5b0 pc=0x19e4c4e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0042a5550, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc00fcbdde0 sp=0xc00fcbd840 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0042a5550, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc00fcbe0d0 sp=0xc00fcbdde0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0042a5420, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc00fcbe3d0 sp=0xc00fcbe0d0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0042a5400?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc00fcbe578 sp=0xc00fcbe3d0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0042a5350, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc00fcbe790 sp=0xc00fcbe578 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0042a5180, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc00fcbe9e8 sp=0xc00fcbe790 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00fcbeb10 sp=0xc00fcbe9e8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00bfdd550, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc00fcbedd8 sp=0xc00fcbeb10 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00fcbeee8 sp=0xc00fcbedd8 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc0434b9380}, 0x0?, 0xc0067e0000) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00fcbf0e0 sp=0xc00fcbeee8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00bfdd550, {0x624a948, 0xc0434b9380}, 0xc0038a6ba0?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc00fcbf128 sp=0xc00fcbf0e0 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00bfdd550, {0x624a948, 0xc0434b9380}, 0xc0067e0000) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x1fd fp=0xc00fcbf198 sp=0xc00fcbf128 pc=0x16b5a3d github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc006f42028, {0x624a948, 0xc0434b9380}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:228 +0xa5 fp=0xc00fcbf1d0 sp=0xc00fcbf198 pc=0x350dd85 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc006f42000, {{0x624a948, 0xc0434b9380}, 0xc00c335000, 0xc00c4eab78}) github.com/cockroachdb/cockroach/pkg/sql/update.go:197 +0x225 fp=0xc00fcbf230 sp=0xc00fcbf1d0 pc=0x352d9a5 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0027bb938, {{0x624a948?, 0xc0434b9380?}, 0xc00c335000?, 0xc00c4eab78?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc00fcbf288 sp=0xc00fcbf230 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc00fcbf488?, 0x624d578?}, {0x624ceb0, 0xc0027bb938}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc00fcbf338 sp=0xc00fcbf288 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc00fcbf370 sp=0xc00fcbf338 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc00fcbf488, {0x624ceb0?, 0xc0027bb938?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc00fcbf400 sp=0xc00fcbf370 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc00fcbf488, {0x624ceb0, 0xc0027bb938}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc00fcbf450 sp=0xc00fcbf400 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc0434b9380?}, 0xc00c335000?, 0xc00c4eab78?}, {0x624ceb0, 0xc0027bb938}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc00fcbf4d0 sp=0xc00fcbf450 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc0048bcdc0, {0x624a948, 0xc0434b9350}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc00fcbf510 sp=0xc00fcbf4d0 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:122 +0x34 fp=0xc00fcbf538 sp=0xc00fcbf510 pc=0x3224ef4 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc00cd4e6c0?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc00fcbf578 sp=0xc00fcbf538 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc00cd4e6c0, {0x624a8a0?, 0xc004e26000?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:121 +0x70 fp=0xc00fcbf5d0 sp=0xc00fcbf578 pc=0x3224e70 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc00cd4e6c0, {0x624a8a0?, 0xc004e26000?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:728 +0x3a fp=0xc00fcbf610 sp=0xc00fcbf5d0 pc=0x230ec1a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0047e0400, {0x624a8a0, 0xc004e26000}, 0xc00cd4e6c0?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc00fcbf6d0 sp=0xc00fcbf610 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc004e26000?}, 0xc00bfdd550?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc00fcbf770 sp=0xc00fcbf6d0 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc004773a70}, 0xc00c359b30, 0xc00bfdd550, 0xc05a061200, 0xc00e342e00, 0xc00c4eaeb0, 0xc011ce9c70) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc00fcc0020 sp=0xc00fcbf770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc004773a70}, 0xe8?, 0xc00c359b30, 0xaa011642efb84501?, {{0x624ceb0?, 0xc0027bb938?}, 0x0?}, 0xc00e342e00, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc00fcc00e0 sp=0xc00fcc0020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc004773a70}, 0xc00c4eaeb0, 0xc00c4eab78?, 0xc00c4eab78, 0xc00e342e00, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc00fcc01f8 sp=0xc00fcc00e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00c4ea600, {0x624a948?, 0xc004773a70}, 0xc00c4eab78, 0xc004773a70?, {0x7f275f329680?, 0xc00d8c32c8}, 0x6e?, 0xc00bbb2138) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc00fcc0300 sp=0xc00fcc01f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00c4ea600, {0x624a8a0, 0xc00b711640}, 0xc00c4eab78, {0x7f275f329680, 0xc00d8c32c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc00fcc0720 sp=0xc00fcc0300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00c4ea600, {0x624a948?, 0xc004773230?}, {{0x6274800, 0xc00099af00}, {0xc00db25005, 0x6e}, 0x3, 0x1}, 0xc00f6226c0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc00fcc10b0 sp=0xc00fcc0720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc004773230?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc00fcc1168 sp=0xc00fcc10b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc004773230?}, {0x6274800?, 0xc00099af00?}, 0xc00db25000?, 0x203016?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc00fcc12b8 sp=0xc00fcc1168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00c4ea600, {0x624a948, 0xc004773230}, {{0x6274800, 0xc00099af00}, {0xc00db25005, 0x6e}, 0x3, 0x1}, 0xc00f6226c0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc00fcc1430 sp=0xc00fcc12b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00c4ea600, {0x624a948, 0xc004773230}, {0xc00f6226c0, {0xc004773320, 0x3, 0x3}, {0x9c5f2e8, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc00fcc1550 sp=0xc00fcc1430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc004134df9, 0x0}, 0x0, {0xc1116e8192a540bc, 0x7fe8946f0b, 0x0}, 0x1}, 0xc00c4ea600, 0xc00fcc18c0, 0xc00fcc18b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc00fcc1708 sp=0xc00fcc1550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00c4ea600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc00fcc1cb8 sp=0xc00fcc1708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00c4ea600, {0x624a8a0, 0xc006d39f80}, 0x4119dce800000000?, 0xc00e9e1dc0?, 0xc0009985c0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00fcc1d70 sp=0xc00fcc1cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc006d39f80?}, {0xc0059fa000?}, 0x3?, 0xc0067e63a8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00fcc1e20 sp=0xc00fcc1d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00fcc1fe0 sp=0xc00fcc1e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fcc1fe8 sp=0xc00fcc1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 120663 [select]: runtime.gopark(0xc0034f9a08?, 0x2?, 0xc0?, 0x8?, 0xc0034f99ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0034f9870 sp=0xc0034f9850 pc=0x4907d6 runtime.selectgo(0xc0034f9a08, 0xc0034f99e8, 0x16e9c57?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0034f99b0 sp=0xc0034f9870 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc016d7e190, {0xc0080f01a8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc0034f9a38 sp=0xc0034f99b0 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc016d7e190, {0xc0080f01a8?, 0xc00a12b098?, 0xc0034f9a80?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc0034f9a78 sp=0xc0034f9a38 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00cf8ec00, {0xc0080f01a8?, 0xc0034f9af8?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc0034f9ac0 sp=0xc0034f9a78 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc00cf8ec00}, {0xc0080f01a8, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc0034f9b08 sp=0xc0034f9ac0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc01250e6c0, {0xc0080f01a8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc0034f9b48 sp=0xc0034f9b08 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc0080f0198, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc0034f9bc8 sp=0xc0034f9b48 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x0?, 0xc01250e6c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc0034f9c60 sp=0xc0034f9bc8 pc=0xc19546 google.golang.org/grpc.recv(0x1?, {0x7f275f301038, 0xc0000af170}, 0x0?, {0x0?, 0x0?}, {0x512da60, 0xc04384e2a0}, 0x0?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc0034f9cd8 sp=0xc0034f9c60 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc0172daf70, {0x512da60?, 0xc04384e2a0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc0034f9e08 sp=0xc0034f9cd8 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x4618e7?, {0x512da60?, 0xc04384e2a0?}) :1 +0x34 fp=0xc0034f9e30 sp=0xc0034f9e08 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc012390f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:263 +0x4c fp=0xc0034f9e60 sp=0xc0034f9e30 pc=0x1d4b66c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:91 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x628a958?, 0xc012390f60}, 0xc000c62700, {0x624a948, 0xc00cf8ee70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:289 +0xa6 fp=0xc0034f9ee0 sp=0xc0034f9e60 pc=0x1d82d66 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x624a948?, 0xc00cf8ee70?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:308 +0x45 fp=0xc0034f9f30 sp=0xc0034f9ee0 pc=0x1d82c65 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0034f9fe0 sp=0xc0034f9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0034f9fe8 sp=0xc0034f9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 428331 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1d3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c88af58 sp=0xc00c88af38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f2fffe8?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00c88af90 sp=0xc00c88af58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f2fff00, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00c88afb0 sp=0xc00c88af90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0122d4d80?, 0xc003369000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00c88afd8 sp=0xc00c88afb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0122d4d80, {0xc003369000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00c88b058 sp=0xc00c88afd8 pc=0x535efa net.(*netFD).Read(0xc0122d4d80, {0xc003369000?, 0x7b5a68?, 0xc0122d4d80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00c88b0a0 sp=0xc00c88b058 pc=0x7a20c9 net.(*conn).Read(0xc0137e3d20, {0xc003369000?, 0x164290f?, 0xc0137e3d20?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00c88b0e8 sp=0xc00c88b0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc003d2d020, {0xc003369000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00c88b130 sp=0xc00c88b0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc006ffcd80, {0xc003369000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00c88b1d0 sp=0xc00c88b130 pc=0x370f896 bufio.(*Reader).fill(0xc0100d1890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00c88b208 sp=0xc00c88b1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0100d1890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00c88b228 sp=0xc00c88b208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0100d1e68?, {0x62295c8, 0xc0100d1890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00c88b260 sp=0xc00c88b228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0100d1800, 0xc010e21440, 0xc00c88b442, 0xc00c88b444, {0x623b718, 0xc0381d82d0}, 0xc0059fa000, 0xc0150068e0, 0xc00c88b440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00c88b398 sp=0xc00c88b260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0100d1800, {0x624a948, 0xc0031e26f0}, 0xc010e21420, 0xc0059fa000, 0xc0031e2630, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00c88b768 sp=0xc00c88b398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0031e26f0}, {0x628a538, 0xc003d2d020}, {{{0xc004afc038, 0x4}}, 0x1, 0xc0031e2660, 0xc0031e2690, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00c88b8a8 sp=0xc00c88b768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0031e26f0}, {0x628a538?, 0xc003d2d020}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00c88be20 sp=0xc00c88b8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0031e2030?}, {0x628a538, 0xc003d2d020}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00c88beb0 sp=0xc00c88be20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0031e2030}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00c88bf30 sp=0xc00c88beb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c88bfe0 sp=0xc00c88bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c88bfe8 sp=0xc00c88bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498310 [runnable]: syscall.Syscall6(0xc0082e5920?, 0xc00a625b48?, 0xc00a625bc8?, 0x542045?, 0xc0082e5920?, 0xc0138aaffb?, 0x1f03c?) GOROOT/src/syscall/syscall_linux.go:90 +0x36 fp=0xc00a625b50 sp=0xc00a625ac8 pc=0x4d8a36 syscall.Syscall6(0x115, 0x28e, 0x0, 0x300000, 0x3, 0x0, 0x0) :1 +0x45 fp=0xc00a625b98 sp=0xc00a625b50 pc=0x4d8de5 golang.org/x/sys/unix.SyncFileRange(0xc00a625c70?, 0xc00a625c78?, 0xc00a625c50?, 0x1058d47?) golang.org/x/sys/unix/external/org_golang_x_sys/unix/zsyscall_linux_amd64.go:424 +0x3f fp=0xc00a625bf8 sp=0xc00a625b98 pc=0xb5173f github.com/cockroachdb/pebble/vfs.(*linuxFile).SyncTo(0x0?, 0x0?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/default_linux.go:95 +0x65 fp=0xc00a625c28 sp=0xc00a625bf8 pc=0x1056925 github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).SyncTo.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:158 +0x3b fp=0xc00a625c58 sp=0xc00a625c28 pc=0x1058c1b github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).timeDiskOp(0xc005f85900, 0xc00a625cc8) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:175 +0xd8 fp=0xc00a625ca0 sp=0xc00a625c58 pc=0x1058d38 github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).SyncTo(0x44532e6?, 0xc0159a6920?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:157 +0x65 fp=0xc00a625d00 sp=0xc00a625ca0 pc=0x1058ba5 github.com/cockroachdb/pebble/vfs.(*enospcFile).SyncTo(0xc007952cc0?, 0xc0138aaffb?) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:413 +0x22 fp=0xc00a625d20 sp=0xc00a625d00 pc=0x10580c2 github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl.(*encryptedFile).SyncTo(0x3f005?, 0x105f79f?) :1 +0x2a fp=0xc00a625d40 sp=0xc00a625d20 pc=0x4457e6a github.com/cockroachdb/pebble/vfs.(*syncingFile).maybeSync(0xc011958780) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/syncing_file.go:151 +0x134 fp=0xc00a625d98 sp=0xc00a625d40 pc=0x105fa74 github.com/cockroachdb/pebble/vfs.(*syncingFile).Write(0xc011958780, {0xc0138aaffb, 0x1f03c, 0x3f005}) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/syncing_file.go:73 +0xff fp=0xc00a625de8 sp=0xc00a625d98 pc=0x105f79f github.com/cockroachdb/pebble.(*compactionFile).Write(0xc01106f880, {0xc0138aaffb?, 0x10a583c?, 0xc0005ee570?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:271 +0x2e fp=0xc00a625e18 sp=0xc00a625de8 pc=0x10f5d8e bufio.(*Writer).Write(0xc00e678180, {0xc0138aa000?, 0x40?, 0xc011a43680?}) GOROOT/src/bufio/bufio.go:669 +0xed fp=0xc00a625e78 sp=0xc00a625e18 pc=0x5b040d github.com/cockroachdb/pebble/sstable.(*Writer).writeCompressedBlock(0xc003b45500, {0xc0138aa000?, 0x20037, 0x40000?}, {0xc001419680, 0x4b1c940?, 0x78}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:1486 +0x98 fp=0xc00a625eb0 sp=0xc00a625e78 pc=0x10dff18 github.com/cockroachdb/pebble/sstable.(*writeQueue).performWrite(0xc00e678140, 0xc003736310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:68 +0x85 fp=0xc00a625f90 sp=0xc00a625eb0 pc=0x10d76c5 github.com/cockroachdb/pebble/sstable.(*writeQueue).runWorker(0xc00e678140) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:109 +0x85 fp=0xc00a625fc8 sp=0xc00a625f90 pc=0x10d7d45 github.com/cockroachdb/pebble/sstable.newWriteQueue.func1() github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0x26 fp=0xc00a625fe0 sp=0xc00a625fc8 pc=0x10d7606 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a625fe8 sp=0xc00a625fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/sstable.newWriteQueue github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0xd6 goroutine 498402 [select]: runtime.gopark(0xc0127d9208?, 0x2?, 0xb8?, 0xa8?, 0xc0127d9204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0127d9090 sp=0xc0127d9070 pc=0x4907d6 runtime.selectgo(0xc0127d9208, 0xc0127d9200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0127d91d0 sp=0xc0127d9090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc003562240) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0127d9238 sp=0xc0127d91d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ef82210, {0x51b9a20?, 0xc00418e780}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0127d9340 sp=0xc0127d9238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0127d9370 sp=0xc0127d9340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0065f9e60, 0xc0127d9448, 0xc0127d9418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0127d93c8 sp=0xc0127d9370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0065f9e60, {0x51b9a20?, 0xc00418e780?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0127d9478 sp=0xc0127d93c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc001709aa0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc01068ec30}, {0x51b9a20, 0xc00418e780}, 0x0?, {0xc00a393c00, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0127d94e0 sp=0xc0127d9478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc001709aa0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc01068ec30}, {0x51b9a20, 0xc00418e780}, 0x2e87cfbae2c21f13?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0127d95d0 sp=0xc0127d94e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc001709aa0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc01068ec30?}, {0x51b9a20?, 0xc00418e780?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc0127d9670 sp=0xc0127d95d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e3aa8, {0x624a948, 0xc001709aa0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0127d96f0 sp=0xc0127d9670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e3aa8?}}, {0x624a948, 0xc001709aa0}, 0xc01068ec30, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0127d97c8 sp=0xc0127d96f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xb0fb830?, {0x624a948?, 0xc001709aa0?}, 0x624a948?, {0x0?, 0xc005751d20?, 0xc00b0fb928?}) :1 +0x65 fp=0xc0127d9818 sp=0xc0127d97c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc015beaf00, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0127d9ab8 sp=0xc0127d9818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc015beaf00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12c8ef1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0127d9bf0 sp=0xc0127d9ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12c8ef1, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0127da338 sp=0xc0127d9bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12c8ef1, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0127dac38 sp=0xc0127da338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12c8ef1, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0127db4a0 sp=0xc0127dac38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e12c8ef1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0127db8c0 sp=0xc0127db4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0127db9f8 sp=0xc0127db8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00d93f680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc0127dbc00 sp=0xc0127db9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00d93f680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc0127dbf30 sp=0xc0127dbc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0127dbfe0 sp=0xc0127dbf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0127dbfe8 sp=0xc0127dbfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 461378 [sleep]: runtime.gopark(0x4b85f3deec8?, 0x801985?, 0x7b?, 0x67?, 0x8026af5fcf?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003eaedf0 sp=0xc003eaedd0 pc=0x4907d6 time.Sleep(0x12383da4b) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003eaee30 sp=0xc003eaedf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0082e72c0, 0x801985, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003eaee88 sp=0xc003eaee30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0082e72c0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003eaeec8 sp=0xc003eaee88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00d9b6000?, 0xf, 0xc00df4bfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003eaefa8 sp=0xc003eaeec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003eaefe0 sp=0xc003eaefa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003eaefe8 sp=0xc003eaefe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 391310 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x21e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a834f58 sp=0xc00a834f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbdbb0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00a834f90 sp=0xc00a834f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbdac8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00a834fb0 sp=0xc00a834f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc011bfa800?, 0xc004a8b000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00a834fd8 sp=0xc00a834fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011bfa800, {0xc004a8b000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00a835058 sp=0xc00a834fd8 pc=0x535efa net.(*netFD).Read(0xc011bfa800, {0xc004a8b000?, 0x7b5a68?, 0xc011bfa800?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00a8350a0 sp=0xc00a835058 pc=0x7a20c9 net.(*conn).Read(0xc005656b10, {0xc004a8b000?, 0x164290f?, 0xc005656b10?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00a8350e8 sp=0xc00a8350a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc013898c60, {0xc004a8b000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00a835130 sp=0xc00a8350e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc011f8aa20, {0xc004a8b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00a8351d0 sp=0xc00a835130 pc=0x370f896 bufio.(*Reader).fill(0xc00dd15090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00a835208 sp=0xc00a8351d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00dd15090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00a835228 sp=0xc00a835208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00dd15668?, {0x62295c8, 0xc00dd15090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00a835260 sp=0xc00a835228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00dd15000, 0xc0468f9580, 0xc00a835442, 0xc00a835444, {0x623b718, 0xc00ce1e630}, 0xc0059fa000, 0xc0049e82c4, 0xc00a835440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00a835398 sp=0xc00a835260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00dd15000, {0x624a948, 0xc01b04f110}, 0xc0468f9570, 0xc0059fa000, 0xc01b04f050, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00a835768 sp=0xc00a835398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc01b04f110}, {0x628a538, 0xc013898c60}, {{{0xc00a3a9009, 0x4}}, 0x1, 0xc01b04f080, 0xc01b04f0b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00a8358a8 sp=0xc00a835768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc01b04f110}, {0x628a538?, 0xc013898c60}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00a835e20 sp=0xc00a8358a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc01b04ef90?}, {0x628a538, 0xc013898c60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00a835eb0 sp=0xc00a835e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc01b04ef90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00a835f30 sp=0xc00a835eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a835fe0 sp=0xc00a835f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a835fe8 sp=0xc00a835fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49140 [select]: runtime.gopark(0xc00ddd4f70?, 0x4?, 0x6?, 0x0?, 0xc00ddd4ed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ddd4d18 sp=0xc00ddd4cf8 pc=0x4907d6 runtime.selectgo(0xc00ddd4f70, 0xc00ddd4ec8, 0xc000acee70?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ddd4e58 sp=0xc00ddd4d18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0035c51e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc00ddd4fc8 sp=0xc00ddd4e58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc00ddd4fe0 sp=0xc00ddd4fc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ddd4fe8 sp=0xc00ddd4fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 45303 [select]: runtime.gopark(0xc001607f00?, 0x2?, 0x73?, 0x0?, 0xc001607ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001607d60 sp=0xc001607d40 pc=0x4907d6 runtime.selectgo(0xc001607f00, 0xc001607ed0, 0x7b55a5?, 0x0, 0xc00e1fc000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001607ea0 sp=0xc001607d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc007cfd630, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc001607f30 sp=0xc001607ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc007cfed20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc001607f88 sp=0xc001607f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc001607fe0 sp=0xc001607f88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001607fe8 sp=0xc001607fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 498416 [runnable]: runtime.gopark(0xae1b6df01c1?, 0x0?, 0x20?, 0x56?, 0xc014079210?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e16f340 sp=0xc00e16f320 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee7fc, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00e16f3a8 sp=0xc00e16f340 pc=0x4a17de sync.runtime_SemacquireMutex(0x45f3b6?, 0x40?, 0x527ed80?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00e16f3d8 sp=0xc00e16f3a8 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee7e0, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00e16f430 sp=0xc00e16f3d8 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0xc001f282c0?, 0x0?, 0xc006771800?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00e16f478 sp=0xc00e16f430 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0019a2000, {0x117c93b?, 0x7f258050dc40?}, 0x0, 0x0, 0xc014079210) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00e16f578 sp=0xc00e16f478 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2727 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc006771800, {0xc008b16fa0, 0x1e, 0x50}, {0xc007fdbc20, 0x1e, 0x1e}, 0x50?, 0x9b?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:847 +0xf8 fp=0xc00e16f5e0 sp=0xc00e16f578 pc=0x10c5078 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc014079a00?, {0xc008b16fa0?, 0x1?, 0x1e?}, {0xc007fdbc20?, 0x4baefa?, 0x6c676c68666f6f54?}, 0x55?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00e16f630 sp=0xc00e16f5e0 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc014079a00, {0xc008b16fa0, 0x1e, 0x50}, {0xc007fdbc20, 0x1e, 0x1e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00e16f680 sp=0xc00e16f630 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc014079420, {0xc007fdbc20?, 0x784f5a4d61684241?, 0x1000000000000?}, 0x7f275fc49310?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00e16f7b0 sp=0xc00e16f680 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc014079420, {0xc008b16fa0?, 0x0?, 0x63484a7268764d70?}, {0xc007fdbc20?, 0x4a59d2?, 0x1e?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00e16f7f0 sp=0xc00e16f7b0 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc014079288, {0xc008b16fa0?, 0x0?, 0x1e?}, {0xc007fdbc20, 0x1e, 0x1e}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00e16f848 sp=0xc00e16f7f0 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc014078f00, {0xc007fdbc20, 0x1e, 0x1e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00e16f8e0 sp=0xc00e16f848 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc004ed4c28, {{0xc007fdbc00, 0x1d, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00e16f930 sp=0xc00e16f8e0 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00d1e0480, {0x624a948, 0xc011b13bf0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc00e16f998 sp=0xc00e16f930 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc011b13bf0}, {0x62b4fe0, 0xc004ed4c28}, {0xc007fdbc00, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc00e16ff68 sp=0xc00e16f998 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc011b13bf0}, {0x628f6b0?, 0xc004ed4c00?}, {0xc007fdbc00, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc00e1700e8 sp=0xc00e16ff68 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc011b13bf0?}, {0x628f6b0?, 0xc004ed4c00?}, {0xc007fdbc00?, 0xc00e170288?, 0x4618e7?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc00e1701d8 sp=0xc00e1700e8 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0?, {0x624a948, 0xc011b13bf0}, {0x628f6b0, 0xc004ed4c00}, {0x10, 0xd3, 0x82, 0xe5, 0x6b, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:95 +0xd3 fp=0xc00e170298 sp=0xc00e1701d8 pc=0x1c96133 github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0x10, 0xd3, 0x82, 0xe5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:2034 +0xbc fp=0xc00e170480 sp=0xc00e170298 pc=0x1d8fb1c github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc010616b40, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:206 +0x36f fp=0xc00e1722f0 sp=0xc00e170480 pc=0x1dc43af github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0126d66e0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00e172940 sp=0xc00e1722f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0126d66e0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00e172e18 sp=0xc00e172940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76f2cec567, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00e173c98 sp=0xc00e172e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc002125400?, {0x624a948, 0xc011b13bf0}, {0xc0040c6d80, 0x8}, 0xc010616b40, 0x8?, 0x0?, {{0x17600a76f2cec567, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00e174918 sp=0xc00e173c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc011b13bf0}, {0xc0040c6d80, 0x8}, 0xc010616b40, 0x400000004?, 0xc00e175968, {{0x17600a76f2cec567, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00e1749d0 sp=0xc00e174918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc003861800, {0x624a948, 0xc011b13bf0}, 0xc010616b40, 0xc0126d66e0, 0xc00e175968, {{0x17600a76f2cec567, 0x0, 0x0}, {0x17600a76d68747e8, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00e175398 sp=0xc00e1749d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc011b13bf0}, 0xc010616b40, 0xc0126d66e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00e175b08 sp=0xc00e175398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc011b13bf0}, 0xc010616b40, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00e175e28 sp=0xc00e175b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76d5016067, 0x0, 0x0}, 0x0, {0x17600a76e458c67b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00e1761f0 sp=0xc00e175e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76d5016067, 0x0, 0x0}, 0x0, {0x17600a76e458c67b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00e176b78 sp=0xc00e1761f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x4263537962514d62?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e458c67b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00e176dc0 sp=0xc00e176b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc011b13b90}, {0x516166464a625851?}, 0xc010616a50) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00e177430 sp=0xc00e176dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc011b13ad0}, 0xc010616a50) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00e1774d8 sp=0xc00e177430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc011b13ad0}, {0x52f0080?, 0xc010616a50}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00e177518 sp=0xc00e1774d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc011b13ad0}, {0x52f0080, 0xc010616a50}, 0xc00a843ca0, 0xc005bc1938) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00e177848 sp=0xc00e177518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc011b13ad0?}, {0x52f0080?, 0xc010616a50?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00e177888 sp=0xc00e177848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc011b13ad0}, {0x52f0080, 0xc010616a50}, 0xc00e177920?, 0xc001f29b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00e1778b8 sp=0xc00e177888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc011b13ad0?}, {0x52f0080?, 0xc010616a50?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e1778f8 sp=0xc00e1778b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc011b13ad0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00e177930 sp=0xc00e1778f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc011b13ad0}, {0x203002?, 0x40?}, 0xc00e1779e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00e1779a8 sp=0xc00e177930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc011b13ad0?}, {0x52f0080?, 0xc010616a50?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00e177a20 sp=0xc00e1779a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc011b13ad0?}, {0x52f0080?, 0xc010616a50?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e177a60 sp=0xc00e177a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc011b13ad0}, {0x52f0080, 0xc010616a50}, 0xc00a843ca0, 0xc005bc1938) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00e177ab0 sp=0xc00e177a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc011b13ad0}, 0xc011f06420, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00e177b08 sp=0xc00e177ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc003f74360, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00e177e48 sp=0xc00e177b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc003f74360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00e177f68 sp=0xc00e177e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00e177fe0 sp=0xc00e177f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e177fe8 sp=0xc00e177fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 48666 [runnable]: runtime.Gosched(...) GOROOT/src/runtime/proc.go:318 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0155fd860) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:570 +0x1f6 fp=0xc00b3e3f88 sp=0xc00b3e3f30 pc=0xbd7156 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc00b3e3fe0 sp=0xc00b3e3f88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b3e3fe8 sp=0xc00b3e3fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 498352 [select]: runtime.gopark(0xc010aeb208?, 0x2?, 0x30?, 0xa6?, 0xc010aeb204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010aeb090 sp=0xc010aeb070 pc=0x4907d6 runtime.selectgo(0xc010aeb208, 0xc010aeb200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc010aeb1d0 sp=0xc010aeb090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0065f98c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc010aeb238 sp=0xc010aeb1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ef82000, {0x51b9a20?, 0xc00418e680}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc010aeb340 sp=0xc010aeb238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc010aeb370 sp=0xc010aeb340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0065f9680, 0xc010aeb448, 0xc010aeb418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc010aeb3c8 sp=0xc010aeb370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0065f9680, {0x51b9a20?, 0xc00418e680?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc010aeb478 sp=0xc010aeb3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc0017092c0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc01068e870}, {0x51b9a20, 0xc00418e680}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc010aeb4e0 sp=0xc010aeb478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0017092c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc01068e870}, {0x51b9a20, 0xc00418e680}, 0x50d6966ae58a0513?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc010aeb5d0 sp=0xc010aeb4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc0017092c0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc01068e870?}, {0x51b9a20?, 0xc00418e680?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc010aeb670 sp=0xc010aeb5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e3a60, {0x624a948, 0xc0017092c0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc010aeb6f0 sp=0xc010aeb670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e3a60?}}, {0x624a948, 0xc0017092c0}, 0xc01068e870, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc010aeb7c8 sp=0xc010aeb6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xb0fb830?, {0x624a948?, 0xc0017092c0?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc00b0fb928?}) :1 +0x65 fp=0xc010aeb818 sp=0xc010aeb7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00e8541e0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010aebab8 sp=0xc010aeb818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00e8541e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1290c4d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010aebbf0 sp=0xc010aebab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1290c4d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010aec338 sp=0xc010aebbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1290c4d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010aecc38 sp=0xc010aec338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1290c4d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc010aed4a0 sp=0xc010aecc38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1290c4d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc010aed8c0 sp=0xc010aed4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc010aed9f8 sp=0xc010aed8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00d93f5f0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc010aedc00 sp=0xc010aed9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00d93f5f0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc010aedf30 sp=0xc010aedc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010aedfe0 sp=0xc010aedf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010aedfe8 sp=0xc010aedfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 345104 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x20?, 0x0?, 0xc014565640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0145655d0 sp=0xc0145655b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00ed57d10, 0x42f) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc014565618 sp=0xc0145655d0 pc=0x4be30c sync.(*Cond).Wait(0xc00f359800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc014565650 sp=0xc014565618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00f359e68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc014565708 sp=0xc014565650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0046ad300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc014565cb8 sp=0xc014565708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0046ad300, {0x624a8a0, 0xc00ed57d40}, 0x414a035e80000000?, 0xc003fa7dc0?, 0xc047456e20?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc014565d70 sp=0xc014565cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00ed57d40?}, {0xc0059fa000?}, 0x3?, 0xc0038a73c8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc014565e20 sp=0xc014565d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc014565fe0 sp=0xc014565e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc014565fe8 sp=0xc014565fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 391311 [select]: runtime.gopark(0xc01762a508?, 0x2?, 0xa0?, 0xae?, 0xc01762a504?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01762a390 sp=0xc01762a370 pc=0x4907d6 runtime.selectgo(0xc01762a508, 0xc01762a500, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01762a4d0 sp=0xc01762a390 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00cce65a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01762a538 sp=0xc01762a4d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ef83ce0, {0x51b9a20?, 0xc007f9f180}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01762a640 sp=0xc01762a538 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x136?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01762a670 sp=0xc01762a640 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc00cce6240, 0xc01762a748, 0xc01762a718) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01762a6c8 sp=0xc01762a670 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00cce6240, {0x51b9a20?, 0xc007f9f180?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01762a778 sp=0xc01762a6c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00e216630?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc010616000}, {0x51b9a20, 0xc007f9f180}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01762a7e0 sp=0xc01762a778 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00e216630}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc010616000}, {0x51b9a20, 0xc007f9f180}, 0x3c25adad2f5c3563?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01762a8d0 sp=0xc01762a7e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x3af386e888636178?, {0x624a948?, 0xc00e216630?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc010616000?}, {0x51b9a20?, 0xc007f9f180?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01762a970 sp=0xc01762a8d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e3d40, {0x624a948, 0xc00e216630}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01762a9f0 sp=0xc01762a970 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e3d40?}}, {0x624a948, 0xc00e216630}, 0xc010616000, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01762aac8 sp=0xc01762a9f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x11522b30?, {0x624a948?, 0xc00e216630?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc011522c28?}) :1 +0x65 fp=0xc01762ab18 sp=0xc01762aac8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00d41d020, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01762adb8 sp=0xc01762ab18 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00d41d020, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2bc56fa, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01762aef0 sp=0xc01762adb8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2bc56fa, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01762b638 sp=0xc01762aef0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2bc56fa, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01762bf38 sp=0xc01762b638 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2bc56fa, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1403 +0x1393 fp=0xc01762c7a0 sp=0xc01762bf38 pc=0x19c8fb3 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2bc56fa, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01762cbc0 sp=0xc01762c7a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc013a4c638, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc01762ce18 sp=0xc01762cbc0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc013a4c600, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc01762cf40 sp=0xc01762ce18 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc013a4c5d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc01762d1d0 sp=0xc01762cf40 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc013a4c4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc01762d770 sp=0xc01762d1d0 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc013a4c4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc01762da60 sp=0xc01762d770 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc013a4c3a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc01762dd60 sp=0xc01762da60 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc013a4c380?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc01762df08 sp=0xc01762dd60 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc013a4c2d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc01762e120 sp=0xc01762df08 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc013a4c100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc01762e378 sp=0xc01762e120 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc01762e4a0 sp=0xc01762e378 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc031197ad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc01762e768 sp=0xc01762e4a0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc01762e880 sp=0xc01762e768 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00ad14000, {0x624a948, 0xc00e2165d0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc01762eaa0 sp=0xc01762e880 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00ad14000, {0x624a948, 0xc00e2165d0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc01762ed98 sp=0xc01762eaa0 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00fe5d7a0, {0x624a948, 0xc00e2165d0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc01762ee78 sp=0xc01762ed98 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc006913880, {0x624a948, 0xc00e2165d0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc01762f150 sp=0xc01762ee78 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0113706e0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc01762f1a8 sp=0xc01762f150 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc00bbd8dc0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc01762f1c0 sp=0xc01762f1a8 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc00314de00) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc01762f1e8 sp=0xc01762f1c0 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc00314de00) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc01762f200 sp=0xc01762f1e8 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc01762f218 sp=0xc01762f200 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x624a948?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc01762f258 sp=0xc01762f218 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc00314de00) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc01762f290 sp=0xc01762f258 pc=0x29a100c github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*noopProcessor).Next(0xc00d4a6000) github.com/cockroachdb/cockroach/pkg/sql/rowexec/noop.go:89 +0x46 fp=0xc01762f2c8 sp=0xc01762f290 pc=0x2595586 github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc0066b0000, {{0xc00de75000?, 0xc0037e9878?}, 0x624cee8?, 0xc011527390?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc01762f340 sp=0xc01762f2c8 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00755e700, {{0x624a948, 0xc00e216510}, 0xc00de75000, 0xc0037e9878}) github.com/cockroachdb/cockroach/pkg/sql/update.go:164 +0xc4 fp=0xc01762f3a0 sp=0xc01762f340 pc=0x352d844 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc05354a858, {{0x624a948?, 0xc00e216510?}, 0xc00de75000?, 0xc0037e9878?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc01762f3f8 sp=0xc01762f3a0 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc0115275f8?, 0x624d578?}, {0x624ceb0, 0xc05354a858}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc01762f4a8 sp=0xc01762f3f8 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc01762f4e0 sp=0xc01762f4a8 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0115275f8, {0x624ceb0?, 0xc05354a858?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc01762f570 sp=0xc01762f4e0 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0115275f8, {0x624ceb0, 0xc05354a858}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc01762f5c0 sp=0xc01762f570 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc00e216510?}, 0xc00de75000?, 0xc0037e9878?}, {0x624ceb0, 0xc05354a858}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc01762f640 sp=0xc01762f5c0 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00791f600, {0x624a948, 0xc00e2164e0}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc01762f680 sp=0xc01762f640 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc005ad8000, {0x624a948, 0xc00e216270}) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:187 +0xea fp=0xc01762f6b8 sp=0xc01762f680 pc=0x299d40a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:247 +0x2e fp=0xc01762f6e0 sp=0xc01762f6b8 pc=0x322578e github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc004e9ee00?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc01762f720 sp=0xc01762f6e0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init(0xc00f478a80?, {0x624a948?, 0xc00e216270?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:246 +0x4f fp=0xc01762f758 sp=0xc01762f720 pc=0x322572f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc008d6e9a0, {0x624a8a0?, 0xc00bbd8d80?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:291 +0x28d fp=0xc01762f8e8 sp=0xc01762f758 pc=0x3225b8d github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc00f6fa2d0, {0x624a8a0, 0xc00bbd8d80}, 0xc031197ad0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc01762f988 sp=0xc01762f8e8 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc001d15380}, 0xc00a60f950, 0xc031197ad0, 0xc007f9ed80, 0xc00685ce00, 0xc0037e9bb0, 0xc010cbf580) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc017630238 sp=0xc01762f988 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc001d15380}, 0xe8?, 0xc00a60f950, 0xae4aae68cffe3c01?, {{0x624ceb0?, 0xc05354a858?}, 0x0?}, 0xc00685ce00, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0176302f8 sp=0xc017630238 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc001d15380}, 0xc0037e9bb0, 0xc0037e9878?, 0xc0037e9878, 0xc00685ce00, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc017630410 sp=0xc0176302f8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0037e9300, {0x624a948?, 0xc001d15380}, 0xc0037e9878, 0xc001d15380?, {0x7f275f329680?, 0xc00dd156c8}, 0x48d?, 0xc00eb2c678) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc017630518 sp=0xc017630410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0037e9300, {0x624a8a0, 0xc00ebbfc00}, 0xc0037e9878, {0x7f275f329680, 0xc00dd156c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc017630938 sp=0xc017630518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0037e9300, {0x624a948?, 0xc013d35740?}, {{0x6274800, 0xc007f9e900}, {0xc002cae006, 0x5d0}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0176312c8 sp=0xc017630938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc013d35740?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc017631380 sp=0xc0176312c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc013d35740?}, {0x6274800?, 0xc007f9e900?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0176314d0 sp=0xc017631380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0037e9300, {0x624a948, 0xc013d35740}, {{0x6274800, 0xc007f9e900}, {0xc002cae006, 0x5d0}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc017631648 sp=0xc0176314d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6274800, 0xc007f9e900}, {0xc002cae006, 0x5d0}, 0x0, 0x1}, {0xc1116e81957e8514, 0x7feb6db37e, 0x0}, {0xc1116e81957e9242, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc017631708 sp=0xc017631648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0037e9300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc017631cb8 sp=0xc017631708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0037e9300, {0x624a8a0, 0xc00f9f2b40}, 0x41386dd000000000?, 0xc001159dc0?, 0xc0468f95b0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc017631d70 sp=0xc017631cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00f9f2b40?}, {0xc0059fa000?}, 0x3?, 0xc006774888?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc017631e20 sp=0xc017631d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc017631fe0 sp=0xc017631e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc017631fe8 sp=0xc017631fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 423632 [select]: runtime.gopark(0xc010572d58?, 0x2?, 0x30?, 0xa0?, 0xc010572d54?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010572be0 sp=0xc010572bc0 pc=0x4907d6 runtime.selectgo(0xc010572d58, 0xc010572d50, 0x10?, 0x0, 0x901?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc010572d20 sp=0xc010572be0 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc015390480) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc010572d88 sp=0xc010572d20 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0037b24d0, {0x51b9a20?, 0xc011420400}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc010572e90 sp=0xc010572d88 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x821?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc010572ec0 sp=0xc010572e90 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc015390000, 0xc010572f98, 0xc010572f68) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc010572f18 sp=0xc010572ec0 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc015390000, {0x51b9a20?, 0xc011420400?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc010572fc8 sp=0xc010572f18 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc010eda630?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc010616d20}, {0x51b9a20, 0xc011420400}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc010573030 sp=0xc010572fc8 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc010eda630}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc010616d20}, {0x51b9a20, 0xc011420400}, 0x4728faf90f461bf5?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc010573120 sp=0xc010573030 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x382883fb17b9385b?, {0x624a948?, 0xc010eda630?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc010616d20?}, {0x51b9a20?, 0xc011420400?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc0105731c0 sp=0xc010573120 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc046762388, {0x624a948, 0xc010eda630}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc010573240 sp=0xc0105731c0 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc046762388?}}, {0x624a948, 0xc010eda630}, 0xc010616d20, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc010573318 sp=0xc010573240 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x10573380?, {0x624a948?, 0xc010eda630?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc010573478?}) :1 +0x65 fp=0xc010573368 sp=0xc010573318 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0083897a0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010573608 sp=0xc010573368 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0083897a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4f00400, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010573740 sp=0xc010573608 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4f00400, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010573e88 sp=0xc010573740 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4f00400, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010574788 sp=0xc010573e88 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4f00400, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc010574ff0 sp=0xc010574788 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4f00400, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc010575410 sp=0xc010574ff0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00d3ee538, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc010575668 sp=0xc010575410 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00d3ee500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc010575790 sp=0xc010575668 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00d3ee4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc010575a20 sp=0xc010575790 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00d3ee3d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc010575fc0 sp=0xc010575a20 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00d3ee3d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc0105762b0 sp=0xc010575fc0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00d3ee2a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc0105765b0 sp=0xc0105762b0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00d3ee280?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc010576758 sp=0xc0105765b0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00d3ee1d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc010576970 sp=0xc010576758 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00d3ee000, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc010576bc8 sp=0xc010576970 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc010576cf0 sp=0xc010576bc8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc01cf2c4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc010576fb8 sp=0xc010576cf0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc0105770c8 sp=0xc010576fb8 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc02b102f60}, 0x456936?, 0xc0088a6000) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc0105772c0 sp=0xc0105770c8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc01cf2c4d0, {0x624a948, 0xc02b102f60}, 0x0?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc010577308 sp=0xc0105772c0 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00a05a728, {0x624a948, 0xc02b102f60}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc010577340 sp=0xc010577308 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00a05a700, {{0x624a948, 0xc02b102f60}, 0xc00bb54400, 0xc007c09878}) github.com/cockroachdb/cockroach/pkg/sql/update.go:197 +0x225 fp=0xc0105773a0 sp=0xc010577340 pc=0x352d9a5 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc04716b260, {{0x624a948?, 0xc02b102f60?}, 0xc00bb54400?, 0xc007c09878?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc0105773f8 sp=0xc0105773a0 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc0105775f8?, 0x624d578?}, {0x624ceb0, 0xc04716b260}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc0105774a8 sp=0xc0105773f8 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc0105774e0 sp=0xc0105774a8 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0105775f8, {0x624ceb0?, 0xc04716b260?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc010577570 sp=0xc0105774e0 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0105775f8, {0x624ceb0, 0xc04716b260}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc0105775c0 sp=0xc010577570 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc02b102f60?}, 0xc00bb54400?, 0xc007c09878?}, {0x624ceb0, 0xc04716b260}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc010577640 sp=0xc0105775c0 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc006870b00, {0x624a948, 0xc02b102f30}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc010577680 sp=0xc010577640 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc0076b7800, {0x624a948, 0xc02b102f00}) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:187 +0xea fp=0xc0105776b8 sp=0xc010577680 pc=0x299d40a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:247 +0x2e fp=0xc0105776e0 sp=0xc0105776b8 pc=0x322578e github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc004bc6700?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc010577720 sp=0xc0105776e0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init(0xc00874a700?, {0x624a948?, 0xc02b102f00?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:246 +0x4f fp=0xc010577758 sp=0xc010577720 pc=0x322572f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc01cf2c0b0, {0x624a8a0?, 0xc00498f8c0?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:291 +0x28d fp=0xc0105778e8 sp=0xc010577758 pc=0x3225b8d github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc035597f90, {0x624a8a0, 0xc00498f8c0}, 0xc01cf2c4d0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc010577988 sp=0xc0105778e8 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc02b102480}, 0xc01725a2d0, 0xc01cf2c4d0, 0xc013ea0380, 0xc003297880, 0xc007c09bb0, 0xc014349260) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc010578238 sp=0xc010577988 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc02b102480}, 0xe8?, 0xc01725a2d0, 0x84fdf2bc2863cc01?, {{0x624ceb0?, 0xc04716b260?}, 0x0?}, 0xc003297880, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0105782f8 sp=0xc010578238 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc02b102480}, 0xc007c09bb0, 0xc007c09878?, 0xc007c09878, 0xc003297880, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc010578410 sp=0xc0105782f8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc007c09300, {0x624a948?, 0xc02b102480}, 0xc007c09878, 0xc02b102480?, {0x7f275f329680?, 0xc0118e6ac8}, 0x535?, 0xc0085a7408) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc010578518 sp=0xc010578410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc007c09300, {0x624a8a0, 0xc00f985540}, 0xc007c09878, {0x7f275f329680, 0xc0118e6ac8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc010578938 sp=0xc010578518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc007c09300, {0x624a948?, 0xc003e2bb60?}, {{0x6274800, 0xc01302be80}, {0xc00ccf335c, 0x6b7}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0105792c8 sp=0xc010578938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc003e2bb60?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc010579380 sp=0xc0105792c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc003e2bb60?}, {0x6274800?, 0xc01302be80?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0105794d0 sp=0xc010579380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc007c09300, {0x624a948, 0xc003e2bb60}, {{0x6274800, 0xc01302be80}, {0xc00ccf335c, 0x6b7}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc010579648 sp=0xc0105794d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6274800, 0xc01302be80}, {0xc00ccf335c, 0x6b7}, 0x0, 0x1}, {0xc1116e818d868acc, 0x7fe375b91f, 0x0}, {0xc1116e818d869373, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc010579708 sp=0xc010579648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc007c09300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc010579cb8 sp=0xc010579708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc007c09300, {0x624a8a0, 0xc00d568900}, 0x4114fc4400000000?, 0xc003dd3dc0?, 0xc0143e93b0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc010579d70 sp=0xc010579cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00d568900?}, {0xc0059fa000?}, 0x3?, 0xc006ffadb0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc010579e20 sp=0xc010579d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc010579fe0 sp=0xc010579e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010579fe8 sp=0xc010579fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 473156 [sleep]: runtime.gopark(0x4b92dc5f454?, 0x400eb5?, 0x7f?, 0x5c?, 0x81174a7cdc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b4b5df0 sp=0xc00b4b5dd0 pc=0x4907d6 time.Sleep(0x14596f3b6) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00b4b5e30 sp=0xc00b4b5df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc03d7438f0, 0x400eb5, {0x4618e7?, 0x20300f?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00b4b5e88 sp=0xc00b4b5e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc03d7438f0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00b4b5ec8 sp=0xc00b4b5e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc0055a9400?, 0xe, 0xc00136cf70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00b4b5fa8 sp=0xc00b4b5ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00b4b5fe0 sp=0xc00b4b5fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b4b5fe8 sp=0xc00b4b5fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 49108 [select]: runtime.gopark(0xc006167f00?, 0x2?, 0x88?, 0x7d?, 0xc006167ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b9d9d60 sp=0xc00b9d9d40 pc=0x4907d6 runtime.selectgo(0xc00b9d9f00, 0xc006167ed0, 0x0?, 0x0, 0xc006167ef0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b9d9ea0 sp=0xc00b9d9d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc011e8aaf0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc00b9d9f30 sp=0xc00b9d9ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00d92ec60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc00b9d9f88 sp=0xc00b9d9f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc00b9d9fe0 sp=0xc00b9d9f88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b9d9fe8 sp=0xc00b9d9fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 120470 [select]: runtime.gopark(0xc00136af70?, 0x4?, 0x6?, 0x0?, 0xc00136aed0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00136ad18 sp=0xc00136acf8 pc=0x4907d6 runtime.selectgo(0xc00136af70, 0xc00136aec8, 0xc000acee70?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00136ae58 sp=0xc00136ad18 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00c483040) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x233 fp=0xc00136afc8 sp=0xc00136ae58 pc=0xbf4d73 google.golang.org/grpc/internal/transport.NewServerTransport.func4() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x26 fp=0xc00136afe0 sp=0xc00136afc8 pc=0xbedaa6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00136afe8 sp=0xc00136afe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x1a38 goroutine 498434 [select]: runtime.gopark(0xc01328b208?, 0x2?, 0x20?, 0xf6?, 0xc01328b204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01328b090 sp=0xc01328b070 pc=0x4907d6 runtime.selectgo(0xc01328b208, 0xc01328b200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01328b1d0 sp=0xc01328b090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0264670e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01328b238 sp=0xc01328b1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc02db6bc30, {0x51b9a20?, 0xc0057f9400}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01328b340 sp=0xc01328b238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01328b370 sp=0xc01328b340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc026466d80, 0xc01328b448, 0xc01328b418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01328b3c8 sp=0xc01328b370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc026466d80, {0x51b9a20?, 0xc0057f9400?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01328b478 sp=0xc01328b3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc011e01800?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0107d0e10}, {0x51b9a20, 0xc0057f9400}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01328b4e0 sp=0xc01328b478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011e01800}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0107d0e10}, {0x51b9a20, 0xc0057f9400}, 0xa17e72d3d8e15f4?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01328b5d0 sp=0xc01328b4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc011e01800?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0107d0e10?}, {0x51b9a20?, 0xc0057f9400?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01328b670 sp=0xc01328b5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043276b50, {0x624a948, 0xc011e01800}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01328b6f0 sp=0xc01328b670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043276b50?}}, {0x624a948, 0xc011e01800}, 0xc0107d0e10, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01328b7c8 sp=0xc01328b6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc011e01800?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc01328b818 sp=0xc01328b7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc011798ae0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01328bab8 sp=0xc01328b818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc011798ae0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e19d5ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01328bbf0 sp=0xc01328bab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e19d5ba8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01328c338 sp=0xc01328bbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e19d5ba8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01328cc38 sp=0xc01328c338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e19d5ba8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc01328d4a0 sp=0xc01328cc38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e19d5ba8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01328d8c0 sp=0xc01328d4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc01328d9f8 sp=0xc01328d8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc011e017d0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc01328dc00 sp=0xc01328d9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc011e017d0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc01328df30 sp=0xc01328dc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01328dfe0 sp=0xc01328df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01328dfe8 sp=0xc01328dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 368719 [sleep]: runtime.gopark(0x4b8acc21d07?, 0x4047a?, 0x18?, 0x12?, 0x80883ad46e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006715df0 sp=0xc006715dd0 pc=0x4907d6 time.Sleep(0x1378b2063) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc006715e30 sp=0xc006715df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc015e86750, 0x4047a, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc006715e88 sp=0xc006715e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc015e86750, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc006715ec8 sp=0xc006715e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00d889500?, 0x25, 0xc00df4bfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc006715fa8 sp=0xc006715ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc006715fe0 sp=0xc006715fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006715fe8 sp=0xc006715fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 49082 [select]: runtime.gopark(0xc01d38d950?, 0x4?, 0xb6?, 0x0?, 0xc01d38d8a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01d38d700 sp=0xc01d38d6e0 pc=0x4907d6 runtime.selectgo(0xc01d38d950, 0xc01d38d898, 0xc00eb72ea0?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01d38d840 sp=0xc01d38d700 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000c62700, 0xc015e94b10, {0x628a900, 0xc015e94d70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:434 +0x345 fp=0xc01d38dcd8 sp=0xc01d38d840 pc=0x1d83c25 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x624a948, 0xc00753eae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:601 +0x367 fp=0xc01d38de18 sp=0xc01d38dcd8 pc=0x1d852e7 runtime/pprof.Do({0x624a948?, 0xc00753e450?}, {{0xc003643780?, 0xc0042f3eb8?, 0xc448e6?}}, 0xc0036432c0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc01d38de88 sp=0xc01d38de18 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x624a948, 0xc00753e450}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:607 +0x1c8 fp=0xc01d38df30 sp=0xc01d38de88 pc=0x1d84f28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01d38dfe0 sp=0xc01d38df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01d38dfe8 sp=0xc01d38dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 496286 [sleep]: runtime.gopark(0x4b88dc6ecf3?, 0x801907?, 0x82?, 0x69?, 0x8068172bd7?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001369df0 sp=0xc001369dd0 pc=0x4907d6 time.Sleep(0x13662a9f3) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc001369e30 sp=0xc001369df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc014b796b0, 0x801907, {0x4618e7?, 0x203005?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc001369e88 sp=0xc001369e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc014b796b0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc001369ec8 sp=0xc001369e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00462df80?, {0xc00b4ab860?, 0x4, 0xc00462df70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc001369fa8 sp=0xc001369ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc001369fe0 sp=0xc001369fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001369fe8 sp=0xc001369fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 51428 [select]: runtime.gopark(0xc0039cda10?, 0x2?, 0x67?, 0x2f?, 0xc0039cd9f4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0039cd878 sp=0xc0039cd858 pc=0x4907d6 runtime.selectgo(0xc0039cda10, 0xc0039cd9f0, 0xc002100400?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0039cd9b8 sp=0xc0039cd878 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc008679ea0, {0xc00e9f6910, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc0039cda40 sp=0xc0039cd9b8 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc008679ea0, {0xc00e9f6910?, 0xc0070e8000?, 0xc0039cda88?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc0039cda80 sp=0xc0039cda40 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc007cfa210, {0xc00e9f6910?, 0xc0039cdb00?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc0039cdac8 sp=0xc0039cda80 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc007cfa210}, {0xc00e9f6910, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc0039cdb10 sp=0xc0039cdac8 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc007735320, {0xc00e9f6910, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc0039cdb50 sp=0xc0039cdb10 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc00e9f6900, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc0039cdbd0 sp=0xc0039cdb50 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x1e63756?, 0xc007735320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc0039cdc68 sp=0xc0039cdbd0 pc=0xc19546 google.golang.org/grpc.recv(0x1d1a426?, {0x7f275f301038, 0xc0000af170}, 0x2?, {0x0?, 0x0?}, {0x512ca20, 0xc0082e4ae0}, 0x203002?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc0039cdce0 sp=0xc0039cdc68 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc011296410, {0x512ca20?, 0xc0082e4ae0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc0039cde10 sp=0xc0039cdce0 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x9172438?, {0x512ca20?, 0xc0082e4ae0?}) :1 +0x34 fp=0xc0039cde38 sp=0xc0039cde10 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc0138fdf40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:363 +0x4c fp=0xc0039cde68 sp=0xc0039cde38 pc=0x16c594c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x624a948, 0xc00dc6f3b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 fp=0xc0039cdf30 sp=0xc0039cde68 pc=0x1d1a762 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0039cdfe0 sp=0xc0039cdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0039cdfe8 sp=0xc0039cdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 496326 [sleep]: runtime.gopark(0x4b88f02bcc8?, 0x80195f?, 0x2c?, 0xd3?, 0x806bd1957f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00136cdf0 sp=0xc00136cdd0 pc=0x4907d6 time.Sleep(0x138e1454c) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00136ce30 sp=0xc00136cdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00d5958c0, 0x80195f, {0x4618e7?, 0x203003?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00136ce88 sp=0xc00136ce30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00d5958c0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00136cec8 sp=0xc00136ce88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00bf58000?, 0x3, 0xc006aedf70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00136cfa8 sp=0xc00136cec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00136cfe0 sp=0xc00136cfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00136cfe8 sp=0xc00136cfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 344544 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0xea?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b4b8f58 sp=0xc00b4b8f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f3001c8?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00b4b8f90 sp=0xc00b4b8f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f3000e0, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00b4b8fb0 sp=0xc00b4b8f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00e89e300?, 0xc0260cc000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00b4b8fd8 sp=0xc00b4b8fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00e89e300, {0xc0260cc000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00b4b9058 sp=0xc00b4b8fd8 pc=0x535efa net.(*netFD).Read(0xc00e89e300, {0xc0260cc000?, 0x7b5a68?, 0xc00e89e300?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00b4b90a0 sp=0xc00b4b9058 pc=0x7a20c9 net.(*conn).Read(0xc004081d58, {0xc0260cc000?, 0x164290f?, 0xc004081d58?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00b4b90e8 sp=0xc00b4b90a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0000c9260, {0xc0260cc000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00b4b9130 sp=0xc00b4b90e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00d2e4258, {0xc0260cc000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00b4b91d0 sp=0xc00b4b9130 pc=0x370f896 bufio.(*Reader).fill(0xc00dd12c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00b4b9208 sp=0xc00b4b91d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00dd12c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00b4b9228 sp=0xc00b4b9208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00dd13268?, {0x62295c8, 0xc00dd12c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00b4b9260 sp=0xc00b4b9228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00dd12c00, 0xc00771a450, 0xc00b4b9442, 0xc00b4b9444, {0x623b718, 0xc01983f710}, 0xc0059fa000, 0xc01284c874, 0xc00b4b9440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00b4b9398 sp=0xc00b4b9260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00dd12c00, {0x624a948, 0xc00dc6b260}, 0xc00771a430, 0xc0059fa000, 0xc00dc6aed0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00b4b9768 sp=0xc00b4b9398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc00dc6b260}, {0x628a538, 0xc0000c9260}, {{{0xc0049ad022, 0x4}}, 0x1, 0xc00dc6af00, 0xc00dc6af60, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00b4b98a8 sp=0xc00b4b9768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc00dc6b260}, {0x628a538?, 0xc0000c9260}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00b4b9e20 sp=0xc00b4b98a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00dc6ad80?}, {0x628a538, 0xc0000c9260}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00b4b9eb0 sp=0xc00b4b9e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00dc6ad80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00b4b9f30 sp=0xc00b4b9eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b4b9fe0 sp=0xc00b4b9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b4b9fe8 sp=0xc00b4b9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 120463 [runnable]: runtime.gopark(0xc008080000?, 0xb?, 0x0?, 0x0?, 0x27f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005609aa0 sp=0xc005609a80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x534825?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc005609ad8 sp=0xc005609aa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300770, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc005609af8 sp=0xc005609ad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc013c7c600?, 0xc001448000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc005609b20 sp=0xc005609af8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc013c7c600, {0xc001448000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc005609ba0 sp=0xc005609b20 pc=0x535efa net.(*netFD).Read(0xc013c7c600, {0xc001448000?, 0x459e71?, 0x4d964e0?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc005609be8 sp=0xc005609ba0 pc=0x7a20c9 net.(*conn).Read(0xc0037300c8, {0xc001448000?, 0xb1?, 0xc005609c98?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc005609c30 sp=0xc005609be8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0155fca20, {0xc001448000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc005609c78 sp=0xc005609c30 pc=0x1641f96 bufio.(*Reader).Read(0xc0155fd380, {0xc0052bc120, 0x9, 0x0?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc005609cb0 sp=0xc005609c78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc0155fd380}, {0xc0052bc120, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc005609cf8 sp=0xc005609cb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc0052bc120?, 0x9?, 0x0?}, {0x62052c0?, 0xc0155fd380?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc005609d48 sp=0xc005609cf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0052bc0e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc005609df8 sp=0xc005609d48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01479a4e0, 0x4d1?, 0xc007374160?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc005609f18 sp=0xc005609df8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc01479a4e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc005609f80 sp=0xc005609f18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc005609fe0 sp=0xc005609f80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005609fe8 sp=0xc005609fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 498423 [select]: runtime.gopark(0xc00d6912e8?, 0x3?, 0x4?, 0x0?, 0xc00d690fb2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d690d38 sp=0xc00d690d18 pc=0x4907d6 runtime.selectgo(0xc00d6912e8, 0xc00d690fac, 0x17600a764ae82c1f?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00d690e78 sp=0xc00d690d38 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc008364f30}, 0xc003dffc20, 0xc006d2d720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00d6915e8 sp=0xc00d690e78 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc008364f30}, 0xc003dffc20, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00d691908 sp=0xc00d6915e8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a764ae82c1f, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00d691cd0 sp=0xc00d691908 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a764ae82c1f, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00d692658 sp=0xc00d691cd0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x3?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00d6928a0 sp=0xc00d692658 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc008364ed0}, {0xc00a1cf500?}, 0xc003dffb30) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00d692f10 sp=0xc00d6928a0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc008364e40}, 0xc003dffb30) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00d692fb8 sp=0xc00d692f10 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc008364e40?}, {0x52f0080?, 0xc003dffb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00d692fe8 sp=0xc00d692fb8 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc008364e40}, {0x52f0080, 0xc003dffb30}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00d693318 sp=0xc00d692fe8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc008364e40?}, {0x52f0080?, 0xc003dffb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00d693358 sp=0xc00d693318 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc008364e40}, {0x52f0080, 0xc003dffb30}, 0xc00d1ad3f0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00d693388 sp=0xc00d693358 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc008364e40?}, {0x52f0080?, 0xc003dffb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00d6933c8 sp=0xc00d693388 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc008364e40?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00d693400 sp=0xc00d6933c8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc008364e40}, {0xc002a40360?, 0x624a948?}, 0xc00d1ad4b8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00d693478 sp=0xc00d693400 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc008364e40?}, {0x52f0080?, 0xc003dffb30?}, 0xc0134d2ba0?, 0x52f0080?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00d6934f0 sp=0xc00d693478 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc008364e40?}, {0x52f0080?, 0xc003dffb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00d693530 sp=0xc00d6934f0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc008364e40?}, {0xc008364e40?, 0x4?}, {0x52f0080?, 0xc003dffb30?}, {0x51b9a20?, 0xc011794480}, 0x203002?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00d693570 sp=0xc00d693530 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc008364e40}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dffb30}, {0x51b9a20, 0xc011794480}, 0xc00d1ad6e8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00d693660 sp=0xc00d693570 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc008364e40}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dffb30}, {0x51b9a20, 0xc011794480}, 0xc00d1ad7c8?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00d6936f8 sp=0xc00d693660 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc008364de0}, 0xc003dffb30, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00d693778 sp=0xc00d6936f8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x4?, {0x624a948?, 0xc008364de0?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00d693800 sp=0xc00d693778 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc014460ea0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00d693aa0 sp=0xc00d693800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc014460ea0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00d693bd8 sp=0xc00d693aa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00d694320 sp=0xc00d693bd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00d694c20 sp=0xc00d694320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00d695488 sp=0xc00d694c20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3758964, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00d6958a8 sp=0xc00d695488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0074ffb38, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00d695b00 sp=0xc00d6958a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0074ffb00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00d695c28 sp=0xc00d695b00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc016171ad0}, {0x7f275f30db68, 0xc0074ffb00}, 0xc01322dd40, {0xc00c720a00, 0x20, 0x21}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc00d695e60 sp=0xc00d695c28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc016171ad0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc00d695f30 sp=0xc00d695e60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00d695fe0 sp=0xc00d695f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d695fe8 sp=0xc00d695fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 122118 [select, 6 minutes]: runtime.gopark(0xc007ba1508?, 0x2?, 0x20?, 0xfb?, 0xc007ba14f4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007ba1368 sp=0xc007ba1348 pc=0x4907d6 runtime.selectgo(0xc007ba1508, 0xc007ba14f0, 0xc012b40f90?, 0x0, 0x23?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc007ba14a8 sp=0xc007ba1368 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc012939ef0, {0x624a948, 0xc012b40f90}, 0xc002a40360, {0x628ab10?, 0xc0016c5790}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1b9 fp=0xc007ba1560 sp=0xc007ba14a8 pc=0x1d1a619 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc00742cb40, {0x628ab10, 0xc0016c5790}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x92 fp=0xc007ba15b0 sp=0xc007ba1560 pc=0x1d18f12 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x51002e0?, 0xc00742cb40}, {0x627d570?, 0xc010b93540}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:344 +0x9f fp=0xc007ba15e8 sp=0xc007ba15b0 pc=0x16c585f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x51002e0, 0xc00742cb40}, {0x627dc78?, 0xc012632340?}, 0xc01314ade0, 0x5683fc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc007ba1920 sp=0xc007ba15e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc012632340?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc007ba1960 sp=0xc007ba1920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x51002e0, 0xc00742cb40}, {0x627dc78, 0xc012632340}, 0x1054f90?, 0xc00d98b000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc007ba1990 sp=0xc007ba1960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc012632340?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc007ba19d0 sp=0xc007ba1990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc007ba1a00 sp=0xc007ba19d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc012b40f00}, {0xc007ba1b08?, 0x46159f?}, 0xc007ba1ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc007ba1a78 sp=0xc007ba1a00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x51002e0, 0xc00742cb40}, {0x627dc78?, 0xc012632340?}, 0xc01314ade0, 0xc00d98b000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc007ba1b00 sp=0xc007ba1a78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc012632340?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc007ba1b40 sp=0xc007ba1b00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x51002e0, 0xc00742cb40}, {0x627dc78, 0xc012632340}, 0xc01314ade0, 0x5683fc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc007ba1b90 sp=0xc007ba1b40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc01479a4e0}, 0xc0142dad80, 0xc000acf290, 0x8fc0b80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc007ba1e48 sp=0xc007ba1b90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc01479a4e0}, 0xc0142dad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc007ba1f68 sp=0xc007ba1e48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc007ba1fe0 sp=0xc007ba1f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007ba1fe8 sp=0xc007ba1fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 346117 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1ca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c832f58 sp=0xc00c832f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbd7f0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00c832f90 sp=0xc00c832f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd708, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00c832fb0 sp=0xc00c832f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00d3f5900?, 0xc000ce5000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00c832fd8 sp=0xc00c832fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00d3f5900, {0xc000ce5000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00c833058 sp=0xc00c832fd8 pc=0x535efa net.(*netFD).Read(0xc00d3f5900, {0xc000ce5000?, 0x7b5a68?, 0xc00d3f5900?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00c8330a0 sp=0xc00c833058 pc=0x7a20c9 net.(*conn).Read(0xc0040801b0, {0xc000ce5000?, 0x164290f?, 0xc0040801b0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00c8330e8 sp=0xc00c8330a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0067604e0, {0xc000ce5000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00c833130 sp=0xc00c8330e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00d1c3da0, {0xc000ce5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00c8331d0 sp=0xc00c833130 pc=0x370f896 bufio.(*Reader).fill(0xc00370c090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00c833208 sp=0xc00c8331d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00370c090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00c833228 sp=0xc00c833208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00370c668?, {0x62295c8, 0xc00370c090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00c833260 sp=0xc00c833228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00370c000, 0xc015333080, 0xc00c833442, 0xc00c833444, {0x623b718, 0xc035b8fe60}, 0xc0059fa000, 0xc006a51304, 0xc00c833440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00c833398 sp=0xc00c833260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00370c000, {0x624a948, 0xc017153e90}, 0xc015333070, 0xc0059fa000, 0xc017153920, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00c833768 sp=0xc00c833398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc017153e90}, {0x628a538, 0xc0067604e0}, {{{0xc00a2d5038, 0x4}}, 0x1, 0xc017153c20, 0xc017153ce0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00c8338a8 sp=0xc00c833768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc017153e90}, {0x628a538?, 0xc0067604e0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00c833e20 sp=0xc00c8338a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc017153740?}, {0x628a538, 0xc0067604e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00c833eb0 sp=0xc00c833e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc017153740}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00c833f30 sp=0xc00c833eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c833fe0 sp=0xc00c833f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c833fe8 sp=0xc00c833fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49141 [runnable]: runtime.gopark(0x0?, 0xb?, 0x0?, 0x0?, 0x62?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004d3aaa0 sp=0xc004d3aa80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0xbef6f1?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc004d3aad8 sp=0xc004d3aaa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300e00, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc004d3aaf8 sp=0xc004d3aad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00094fc80?, 0xc0079e0000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc004d3ab20 sp=0xc004d3aaf8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00094fc80, {0xc0079e0000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc004d3aba0 sp=0xc004d3ab20 pc=0x535efa net.(*netFD).Read(0xc00094fc80, {0xc0079e0000?, 0xc004d3ac70?, 0x46159f?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc004d3abe8 sp=0xc004d3aba0 pc=0x7a20c9 net.(*conn).Read(0xc013f35be8, {0xc0079e0000?, 0xc004d3ad48?, 0x18?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc004d3ac30 sp=0xc004d3abe8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00d92eba0, {0xc0079e0000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc004d3ac78 sp=0xc004d3ac30 pc=0x1641f96 bufio.(*Reader).Read(0xc003ca3980, {0xc004cd49e0, 0x9, 0xc005bc1500?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc004d3acb0 sp=0xc004d3ac78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc003ca3980}, {0xc004cd49e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc004d3acf8 sp=0xc004d3acb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004cd49e0?, 0x9?, 0x0?}, {0x62052c0?, 0xc003ca3980?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc004d3ad48 sp=0xc004d3acf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc004cd49a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc004d3adf8 sp=0xc004d3ad48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0035c51e0, 0x6213b80?, 0xc00d77b18c?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc004d3af18 sp=0xc004d3adf8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc0035c51e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc004d3af80 sp=0xc004d3af18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc004d3afe0 sp=0xc004d3af80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004d3afe8 sp=0xc004d3afe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 120495 [select, 6 minutes]: runtime.gopark(0xc017513578?, 0x2?, 0x0?, 0x4?, 0xc01751353c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0175133a8 sp=0xc017513388 pc=0x4907d6 runtime.selectgo(0xc017513578, 0xc017513538, 0xc017ed2640?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0175134e8 sp=0xc0175133a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000c62700, {0x628a958, 0xc017712330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:313 +0x208 fp=0xc0175135b0 sp=0xc0175134e8 pc=0x1d82b08 github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x525f4e0?, 0xc000c62700}, {0x627d570?, 0xc01896d260}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:244 +0x9f fp=0xc0175135e8 sp=0xc0175135b0 pc=0x1d4b57f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc017d48dd0?}, 0xc0183990c8, 0x5683c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc017513920 sp=0xc0175135e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc017d48dd0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc017513960 sp=0xc017513920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc017d48dd0}, 0x1054f90?, 0xc017ed2600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc017513990 sp=0xc017513960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc017d48dd0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc0175139d0 sp=0xc017513990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc017513a00 sp=0xc0175139d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc018348420}, {0xc017513b08?, 0x46159f?}, 0xc017513ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc017513a78 sp=0xc017513a00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc017d48dd0?}, 0xc0183990c8, 0xc017ed2600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc017513b00 sp=0xc017513a78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc017d48dd0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc017513b40 sp=0xc017513b00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc017d48dd0}, 0xc0183990c8, 0x5683c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc017513b90 sp=0xc017513b40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc017ef90e0, 0xc000ace270, 0x8ff1fe0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc017513e48 sp=0xc017513b90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc017ef90e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc017513f68 sp=0xc017513e48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc017513fe0 sp=0xc017513f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc017513fe8 sp=0xc017513fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498043 [select]: runtime.gopark(0xc00be81e60?, 0x3?, 0x0?, 0x0?, 0xc00be81e32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00be81cb8 sp=0xc00be81c98 pc=0x4907d6 runtime.selectgo(0xc00be81e60, 0xc00be81e2c, 0xc00e89d6b0?, 0x0, 0xc005976e18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00be81df8 sp=0xc00be81cb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc024bde850, {0x624a948, 0xc00e89d6b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00be81eb0 sp=0xc00be81df8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc00e89d6b0?}) :1 +0x39 fp=0xc00be81ed8 sp=0xc00be81eb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc00e89d6b0}, {0x53ddbcd?, 0x6294e00?}, 0xc00cf53fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00be81f50 sp=0xc00be81ed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00be81fe0 sp=0xc00be81f50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00be81fe8 sp=0xc00be81fe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 498417 [select]: runtime.gopark(0xc00e185808?, 0x3?, 0x2?, 0x0?, 0xc00e1854d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e185258 sp=0xc00e185238 pc=0x4907d6 runtime.selectgo(0xc00e185808, 0xc00e1854cc, 0x17600a76a98503b2?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00e185398 sp=0xc00e185258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc010edb7d0}, 0xc010616f00, 0xc006d2cf00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00e185b08 sp=0xc00e185398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc010edb7d0}, 0xc010616f00, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00e185e28 sp=0xc00e185b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76a792b03d, 0x0, 0x0}, 0x0, {0x17600a76e49df3d5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00e1861f0 sp=0xc00e185e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76a792b03d, 0x0, 0x0}, 0x0, {0x17600a76e49df3d5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00e186b78 sp=0xc00e1861f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00e186f20?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e49df3d5, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00e186dc0 sp=0xc00e186b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc010edb680}, {0x620f700?}, 0xc010616e10) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00e187430 sp=0xc00e186dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc010edb530}, 0xc010616e10) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00e1874d8 sp=0xc00e187430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc010edb530}, {0x52f0080?, 0xc010616e10}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00e187518 sp=0xc00e1874d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc010edb530}, {0x52f0080, 0xc010616e10}, 0xc003402820, 0xc004fea558) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00e187848 sp=0xc00e187518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc010edb530?}, {0x52f0080?, 0xc010616e10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00e187888 sp=0xc00e187848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc010edb530}, {0x52f0080, 0xc010616e10}, 0xc00e187920?, 0xc003b9d3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00e1878b8 sp=0xc00e187888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc010edb530?}, {0x52f0080?, 0xc010616e10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e1878f8 sp=0xc00e1878b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc010edb530?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00e187930 sp=0xc00e1878f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc010edb530}, {0x203000?, 0x40?}, 0xc00e1879e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00e1879a8 sp=0xc00e187930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc010edb530?}, {0x52f0080?, 0xc010616e10?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00e187a20 sp=0xc00e1879a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc010edb530?}, {0x52f0080?, 0xc010616e10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e187a60 sp=0xc00e187a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc010edb530}, {0x52f0080, 0xc010616e10}, 0xc003402820, 0xc004fea558) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00e187ab0 sp=0xc00e187a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc010edb530}, 0xc011f066c0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00e187b08 sp=0xc00e187ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc015390b40, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00e187e48 sp=0xc00e187b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc015390b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00e187f68 sp=0xc00e187e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00e187fe0 sp=0xc00e187f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e187fe8 sp=0xc00e187fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 11307 [chan receive]: runtime.gopark(0x2faf080?, 0x3b9aca00?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f057910 sp=0xc00f0578f0 pc=0x4907d6 runtime.chanrecv(0xc0084292c0, 0xc00f057b58, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc00f0579a0 sp=0xc00f057910 pc=0x45a91b runtime.chanrecv1(0x1?, 0xc002125400?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc00f0579c8 sp=0xc00f0579a0 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1290 +0x127 fp=0xc00f057b90 sp=0xc00f0579c8 pc=0x19c9a07 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0f5e2b8, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1468 +0x17b9 fp=0xc00f0583f8 sp=0xc00f057b90 pc=0x19c93d9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0f5e2b8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f058818 sp=0xc00f0583f8 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00a8ed138, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00f058a70 sp=0xc00f058818 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00a8ed100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00f058b98 sp=0xc00f058a70 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00a8ed0d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc00f058e28 sp=0xc00f058b98 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00a8ecfd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc00f0593c8 sp=0xc00f058e28 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00a8ecfd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc00f0596b8 sp=0xc00f0593c8 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00a8ecea0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc00f0599b8 sp=0xc00f0596b8 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00a8ece80?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc00f059b60 sp=0xc00f0599b8 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00a8ecdd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc00f059d78 sp=0xc00f059b60 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00a8ecc00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc00f059fd0 sp=0xc00f059d78 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00f05a0f8 sp=0xc00f059fd0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00ec980b0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc00f05a3c0 sp=0xc00f05a0f8 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc00f05a4d8 sp=0xc00f05a3c0 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00bbf6a80, {0x624a948, 0xc012956090}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc00f05a6f8 sp=0xc00f05a4d8 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00bbf6a80, {0x624a948, 0xc012956090}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc00f05a9f0 sp=0xc00f05a6f8 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc008428f00, {0x624a948, 0xc012956090}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc00f05aad0 sp=0xc00f05a9f0 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00b77ee00, {0x624a948, 0xc012956090}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc00f05ada8 sp=0xc00f05aad0 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc014c321e0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc00f05ae00 sp=0xc00f05ada8 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc01573c9c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc00f05ae18 sp=0xc00f05ae00 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc002a08a10) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:202 +0x2e fp=0xc00f05ae68 sp=0xc00f05ae18 pc=0x2860bee github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:48 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc013e76680) github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:195 +0x4a fp=0xc00f05afc8 sp=0xc00f05ae68 pc=0x299be6a github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc008429080) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f fp=0xc00f05b088 sp=0xc00f05afc8 pc=0x293bf5f github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc002de2780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc00f05b0b0 sp=0xc00f05b088 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc002de2780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc00f05b0c8 sp=0xc00f05b0b0 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc00f05b0e0 sp=0xc00f05b0c8 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0106e3c00?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc00f05b120 sp=0xc00f05b0e0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc002de2780) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc00f05b158 sp=0xc00f05b120 pc=0x29a100c github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc003666000, {{0xc0106e3c00?, 0xc0043bd878?}, 0x624cee8?, 0xc00f05b220?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc00f05b1d0 sp=0xc00f05b158 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00571d500, {{0x624a948, 0xc00c8e5ec0}, 0xc0106e3c00, 0xc0043bd878}) github.com/cockroachdb/cockroach/pkg/sql/update.go:164 +0xc4 fp=0xc00f05b230 sp=0xc00f05b1d0 pc=0x352d844 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc01506d128, {{0x624a948?, 0xc00c8e5ec0?}, 0xc0106e3c00?, 0xc0043bd878?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc00f05b288 sp=0xc00f05b230 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc00f05b488?, 0x624d578?}, {0x624ceb0, 0xc01506d128}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc00f05b338 sp=0xc00f05b288 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc00f05b370 sp=0xc00f05b338 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc00f05b488, {0x624ceb0?, 0xc01506d128?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc00f05b400 sp=0xc00f05b370 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc00f05b488, {0x624ceb0, 0xc01506d128}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc00f05b450 sp=0xc00f05b400 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc00c8e5ec0?}, 0xc0106e3c00?, 0xc0043bd878?}, {0x624ceb0, 0xc01506d128}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc00f05b4d0 sp=0xc00f05b450 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00dceb080, {0x624a948, 0xc00c8e5e90}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc00f05b510 sp=0xc00f05b4d0 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:122 +0x34 fp=0xc00f05b538 sp=0xc00f05b510 pc=0x3224ef4 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc018b8b320?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc00f05b578 sp=0xc00f05b538 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc018b8b320, {0x624a8a0?, 0xc01573c980?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:121 +0x70 fp=0xc00f05b5d0 sp=0xc00f05b578 pc=0x3224e70 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc018b8b320, {0x624a8a0?, 0xc01573c980?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:728 +0x3a fp=0xc00f05b610 sp=0xc00f05b5d0 pc=0x230ec1a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00becc200, {0x624a8a0, 0xc01573c980}, 0xc018b8b320?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc00f05b6d0 sp=0xc00f05b610 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc01573c980?}, 0xc00ec980b0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc00f05b770 sp=0xc00f05b6d0 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc00c8e5020}, 0xc0134f1ef0, 0xc00ec980b0, 0xc0178afd80, 0xc00898dc00, 0xc0043bdbb0, 0xc006d04bd0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc00f05c020 sp=0xc00f05b770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc00c8e5020}, 0xe8?, 0xc0134f1ef0, 0xb258211aa26ea01?, {{0x624ceb0?, 0xc01506d128?}, 0x0?}, 0xc00898dc00, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc00f05c0e0 sp=0xc00f05c020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc00c8e5020}, 0xc0043bdbb0, 0xc0043bd878?, 0xc0043bd878, 0xc00898dc00, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc00f05c1f8 sp=0xc00f05c0e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0043bd300, {0x624a948?, 0xc00c8e5020}, 0xc0043bd878, 0xc00c8e5020?, {0x7f275f329680?, 0xc00e8da2c8}, 0x6e?, 0xc010693868) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc00f05c300 sp=0xc00f05c1f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0043bd300, {0x624a8a0, 0xc01573c280}, 0xc0043bd878, {0x7f275f329680, 0xc00e8da2c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc00f05c720 sp=0xc00f05c300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0043bd300, {0x624a948?, 0xc00df6db90?}, {{0x6274800, 0xc00d597b00}, {0xc0036a0005, 0x6e}, 0x3, 0x1}, 0xc005352900, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc00f05d0b0 sp=0xc00f05c720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc00df6db90?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc00f05d168 sp=0xc00f05d0b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc00df6db90?}, {0x6274800?, 0xc00d597b00?}, 0xc0036a0000?, 0x203001?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc00f05d2b8 sp=0xc00f05d168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0043bd300, {0x624a948, 0xc00df6db90}, {{0x6274800, 0xc00d597b00}, {0xc0036a0005, 0x6e}, 0x3, 0x1}, 0xc005352900, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc00f05d430 sp=0xc00f05d2b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0043bd300, {0x624a948, 0xc00df6db90}, {0xc005352900, {0xc00c8e45d0, 0x3, 0x3}, {0x9c5f2e8, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc00f05d550 sp=0xc00f05d430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0042bfa09, 0x0}, 0x0, {0xc1116e8193ce98da, 0x7fe9bdc729, 0x0}, 0x1}, 0xc0043bd300, 0xc00f05d8c0, 0xc00f05d8b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc00f05d708 sp=0xc00f05d550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0043bd300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc00f05dcb8 sp=0xc00f05d708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0043bd300, {0x624a8a0, 0xc004ec93c0}, 0x4118fde400000000?, 0xc003cb5dc0?, 0xc008e74730?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00f05dd70 sp=0xc00f05dcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc004ec93c0?}, {0xc0059fa000?}, 0x3?, 0xc00609d728?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00f05de20 sp=0xc00f05dd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00f05dfe0 sp=0xc00f05de20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f05dfe8 sp=0xc00f05dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 115781 [select, 6 minutes]: runtime.gopark(0xc00cb68e58?, 0x2?, 0x0?, 0x30?, 0xc00cb68dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cb68c80 sp=0xc00cb68c60 pc=0x4907d6 runtime.selectgo(0xc00cb68e58, 0xc00cb68df8, 0xc00c7c5140?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00cb68dc0 sp=0xc00cb68c80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc00c7c5140}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc00cb68f58 sp=0xc00cb68dc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00cb68f78 sp=0xc00cb68f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00cb68fe0 sp=0xc00cb68f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cb68fe8 sp=0xc00cb68fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498446 [select]: runtime.gopark(0xc01115e2d8?, 0x2?, 0xd8?, 0xb5?, 0xc01115e2d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01115e160 sp=0xc01115e140 pc=0x4907d6 runtime.selectgo(0xc01115e2d8, 0xc01115e2d0, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01115e2a0 sp=0xc01115e160 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc006977320) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01115e308 sp=0xc01115e2a0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030826160, {0x51b9a20?, 0xc003165b00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01115e410 sp=0xc01115e308 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcb?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01115e440 sp=0xc01115e410 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0069770e0, 0xc01115e518, 0xc01115e4e8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01115e498 sp=0xc01115e440 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0069770e0, {0x51b9a20?, 0xc003165b00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01115e548 sp=0xc01115e498 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc015357770?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0172263c0}, {0x51b9a20, 0xc003165b00}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01115e5b0 sp=0xc01115e548 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc015357770}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0172263c0}, {0x51b9a20, 0xc003165b00}, 0x5fb80d75b00713?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01115e6a0 sp=0xc01115e5b0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x7e3d8b96002f0b9b?, {0x624a948?, 0xc015357770?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0172263c0?}, {0x51b9a20?, 0xc003165b00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01115e740 sp=0xc01115e6a0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0432771b0, {0x624a948, 0xc015357770}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01115e7c0 sp=0xc01115e740 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0432771b0?}}, {0x624a948, 0xc015357770}, 0xc0172263c0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01115e898 sp=0xc01115e7c0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xa5fe900?, {0x624a948?, 0xc015357770?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc00a5fe9f8?}) :1 +0x65 fp=0xc01115e8e8 sp=0xc01115e898 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00f74c5a0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01115eb88 sp=0xc01115e8e8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00f74c5a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01115ecc0 sp=0xc01115eb88 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01115f408 sp=0xc01115ecc0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01115fd08 sp=0xc01115f408 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x624a948, 0xc015357770}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1506 +0x1bc fp=0xc01115ff30 sp=0xc01115fd08 pc=0x19ca1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01115ffe0 sp=0xc01115ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01115ffe8 sp=0xc01115ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 380327 [semacquire]: runtime.gopark(0xae1b7194945?, 0x0?, 0x0?, 0x0?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01368fc68 sp=0xc01368fc48 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x34?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc01368fcd0 sp=0xc01368fc68 pc=0x4a17de sync.runtime_SemacquireMutex(0x0?, 0x40?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc01368fd00 sp=0xc01368fcd0 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x7f273cbcbde0?, 0xc00cde35c0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc01368fd58 sp=0xc01368fd00 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0x100c0107fc580?, 0x7f27583c84e0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc01368fda0 sp=0xc01368fd58 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0107fc580, {0x17?, 0x17?}, 0x0, 0xc01c712578, 0xc00d99a310) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc01368fea0 sp=0xc01368fda0 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc01c712300, 0xe0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc01368ff40 sp=0xc01368fea0 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc01c712300, {0xc0040c6fe0, 0xf, 0xf}, 0xf?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc01368fff0 sp=0xc01368ff40 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc00e7f3260?, 0x0?, 0x0?}, {0xc0040c6fe0?, 0xf?, 0xc0069086e0?}, 0x30?, 0xba?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc013690058 sp=0xc01368fff0 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc00d99b8e0?, {0xc00e7f3260?, 0x1?, 0xf?}, {0xc0040c6fe0?, 0xc0040c6fe0?, 0x1f?}, 0xf?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc0136900a8 sp=0xc013690058 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00d99b8e0, {0xc00e7f3260, 0xf, 0x21}, {0xc0040c6fe0, 0xf, 0xf}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc0136900f8 sp=0xc0136900a8 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00d99a520, {0xc0040c6fe0?, 0x49e9380?, 0x1?}, 0xc0136902f0?, 0xed?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc013690228 sp=0xc0136900f8 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00d99a520, {0xc00e7f3260?, 0xf?, 0xc008304780?}, {0xc0040c6fe0?, 0xc008304780?, 0xe?}, 0x50?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc013690268 sp=0xc013690228 pc=0x115285d github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00d99a000, {0xc0040c6fe0, 0xf, 0xf}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc013690300 sp=0xc013690268 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0067e1338, {{0xc008304780, 0xe, 0x50}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:373 +0x147 fp=0xc013690358 sp=0xc013690300 pc=0x1863127 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc010edd900, {{0xc00ad67480, 0x8, 0x10}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:520 +0x4d5 fp=0xc013690418 sp=0xc013690358 pc=0x180d135 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0089bb680, {0x624a948, 0xc011fe3da0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc013690480 sp=0xc013690418 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc011fe3da0}, {0x62b4f18, 0xc010edd900}, {0xc00ad67480, 0x8, 0x10}, {0x17600a76b4836c0d, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc013690a50 sp=0xc013690480 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc011fe3da0}, {0x628f760?, 0xc0067e1080?}, {0xc00ad67480, 0x8, 0x10}, {0x17600a76b4836c0d, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc013690bd0 sp=0xc013690a50 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x62cc5e0, 0xc04384ec78}, {{0x17600a76b4836c0d, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x385 fp=0xc013691150 sp=0xc013690bd0 pc=0x1cf2745 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76b4836c0d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc0136919e0 sp=0xc013691150 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00db28c30, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc013693850 sp=0xc0136919e0 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0136942c0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc013693ec0 sp=0xc013693850 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc00328bc00, {0x624a948, 0xc011fe3da0}, 0xc00db28c30, 0xc011044500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc013694b50 sp=0xc013693ec0 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00328bc00, {0x624a948, 0xc011fe3da0}, 0xc00db28c30, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc013694e70 sp=0xc013694b50 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00328bc00, {_, _}, {{{0x17600a76b4836c0d, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc013695238 sp=0xc013694e70 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76b4836c0d, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc013695bc0 sp=0xc013695238 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc013695e08 sp=0xc013695bc0 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc011fe3ce0}, {0x2d3?}, 0xc00db28a50) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc013696478 sp=0xc013695e08 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc011fe3c50}, 0xc00db28a50) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc013696520 sp=0xc013696478 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc011fe3c50?}, {0x52f0080?, 0xc00db28a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc013696550 sp=0xc013696520 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc011fe3c50}, {0x52f0080, 0xc00db28a50}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc013696880 sp=0xc013696550 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011fe3c50?}, {0x52f0080?, 0xc00db28a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0136968c0 sp=0xc013696880 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc011fe3c50}, {0x52f0080, 0xc00db28a50}, 0xc013696958?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0136968f0 sp=0xc0136968c0 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011fe3c50?}, {0x52f0080?, 0xc00db28a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc013696930 sp=0xc0136968f0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc011fe3c50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc013696968 sp=0xc013696930 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc011fe3c50}, {0xc44d59?, 0xc004e9ee00?}, 0xc013696a20) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0136969e0 sp=0xc013696968 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc011fe3c50?}, {0x52f0080?, 0xc00db28a50?}, 0x51b9a20?, 0xc010ef3680?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc013696a58 sp=0xc0136969e0 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011fe3c50?}, {0x52f0080?, 0xc00db28a50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc013696a98 sp=0xc013696a58 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc011fe3c50?}, {0xc011fe3c50?, 0x4?}, {0x52f0080?, 0xc00db28a50?}, {0x51b9a20?, 0xc004a54d80}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc013696ad8 sp=0xc013696a98 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011fe3c50}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00db28a50}, {0x51b9a20, 0xc004a54d80}, 0xc013696c50?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc013696bc8 sp=0xc013696ad8 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc011fe3c50}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00db28a50}, {0x51b9a20, 0xc004a54d80}, 0xc013696d30?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc013696c60 sp=0xc013696bc8 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc011fe3bf0}, 0xc00db28a50, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc013696ce0 sp=0xc013696c60 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc011cb0980?, {0x624a948?, 0xc011fe3bf0?}, 0x624a948?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc013696d68 sp=0xc013696ce0 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00559f080, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc013697008 sp=0xc013696d68 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00559f080, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc013697140 sp=0xc013697008 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc013697888 sp=0xc013697140 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc013698188 sp=0xc013697888 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0136989f0 sp=0xc013698188 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1e2f520, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc013698e10 sp=0xc0136989f0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00ccdb038, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc013699068 sp=0xc013698e10 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00ccdb000, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc013699190 sp=0xc013699068 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00ccdafd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc013699420 sp=0xc013699190 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00ccdaed0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc0136999c0 sp=0xc013699420 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00ccdaed0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc013699cb0 sp=0xc0136999c0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00ccdada0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc013699fb0 sp=0xc013699cb0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00ccdad80?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc01369a158 sp=0xc013699fb0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00ccdacd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc01369a370 sp=0xc01369a158 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00ccdab00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc01369a5c8 sp=0xc01369a370 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc01369a6f0 sp=0xc01369a5c8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00ca69e40, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc01369a9b8 sp=0xc01369a6f0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc01369aad0 sp=0xc01369a9b8 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc003f46000, {0x624a948, 0xc011fe3b90}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc01369acf0 sp=0xc01369aad0 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc003f46000, {0x624a948, 0xc011fe3b90}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc01369afe8 sp=0xc01369acf0 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc010df4540, {0x624a948, 0xc011fe3b90}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc01369b0c8 sp=0xc01369afe8 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00534bc00, {0x624a948, 0xc011fe3b90}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc01369b3a0 sp=0xc01369b0c8 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc000f280a0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc01369b3f8 sp=0xc01369b3a0 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc007305dc0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc01369b410 sp=0xc01369b3f8 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc010df4600) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f fp=0xc01369b4d0 sp=0xc01369b410 pc=0x293bf5f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc00bfdc420) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc01369b4e8 sp=0xc01369b4d0 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc01369b500 sp=0xc01369b4e8 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc00534a380?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc01369b540 sp=0xc01369b500 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc00bfdc420, {0x624a8a0?, 0xc007305bc0?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc01369b6d0 sp=0xc01369b540 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0120f8f00, {0x624a8a0, 0xc007305bc0}, 0xc00ca69e40?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc01369b770 sp=0xc01369b6d0 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc011fe3830}, 0xc00db28780, 0xc00ca69e40, 0xc004a54a80, 0xc00e816700, 0xc0055deeb0, 0xc00d57e9f0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc01369c020 sp=0xc01369b770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc011fe3830}, 0xe8?, 0xc00db28780, 0x5c263de6c825bc01?, {{0x624ce40?, 0xc00d4430e0?}, 0x0?}, 0xc00e816700, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc01369c0e0 sp=0xc01369c020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc011fe3830}, 0xc0055deeb0, 0xc0055deb78?, 0xc0055deb78, 0xc00e816700, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc01369c1f8 sp=0xc01369c0e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0055de600, {0x624a948?, 0xc011fe3830}, 0xc0055deb78, 0xc011fe3830?, {0x7f275f329680?, 0xc003490ac8}, 0xac?, 0xc008b384b8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc01369c300 sp=0xc01369c1f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0055de600, {0x624a8a0, 0xc007305880}, 0xc0055deb78, {0x7f275f329680, 0xc003490ac8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc01369c720 sp=0xc01369c300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0055de600, {0x624a948?, 0xc00ebcdf20?}, {{0x6273700, 0xc012bf8b40}, {0xc0062bb017, 0xd8}, 0x0, 0x1}, 0xc003a2fc20, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc01369d0b0 sp=0xc01369c720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc00ebcdf20?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc01369d168 sp=0xc01369d0b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc00ebcdf20?}, {0x6273700?, 0xc012bf8b40?}, 0x624a948?, 0xc00ebcdf20?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc01369d2b8 sp=0xc01369d168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0055de600, {0x624a948, 0xc00ebcdf20}, {{0x6273700, 0xc012bf8b40}, {0xc0062bb017, 0xd8}, 0x0, 0x1}, 0xc003a2fc20, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc01369d430 sp=0xc01369d2b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0055de600, {0x624a948, 0xc00ebcdf20}, {0xc003a2fc20, {0x9c5fde0, 0x0, 0x0}, {0xc00ad67420, 0x6, 0x6}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc01369d550 sp=0xc01369d430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0062bb12a, 0x0}, 0x0, {0xc1116e8194c2ff0f, 0x7feab22d62, 0x0}, 0x1}, 0xc0055de600, 0xc01369d8c0, 0xc01369d8b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc01369d708 sp=0xc01369d550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0055de600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc01369dcb8 sp=0xc01369d708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0055de600, {0x624a8a0, 0xc00b163cc0}, 0x4118571c00000000?, 0xc004ea7dc0?, 0xc01607dd70?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc01369dd70 sp=0xc01369dcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00b163cc0?}, {0xc0059fa000?}, 0x3?, 0xc003ca4870?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc01369de20 sp=0xc01369dd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc01369dfe0 sp=0xc01369de20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01369dfe8 sp=0xc01369dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498378 [select]: runtime.gopark(0xc006aebc18?, 0x3?, 0xd2?, 0xb9?, 0xc006aebb3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006aeb8f0 sp=0xc006aeb8d0 pc=0x4907d6 runtime.selectgo(0xc006aebc18, 0xc006aebb34, 0xc011cb0080?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc006aeba30 sp=0xc006aeb8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc00729c5a0}, {0x6221560, 0xc038124c60}, {0x46?, {0xc006aebe01?, 0x6aebe60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc006aebda0 sp=0xc006aeba30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc00729c5a0?}, 0x164, 0xc038124c60, 0x1a?, 0xc018974900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc006aebe70 sp=0xc006aebda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc00729c5a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc006aebf30 sp=0xc006aebe70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc006aebfe0 sp=0xc006aebf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006aebfe8 sp=0xc006aebfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498465 [runnable]: runtime.gopark(0xae1b64a8c62?, 0x0?, 0x0?, 0x0?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010e51a48 sp=0xc010e51a28 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc010e51ab0 sp=0xc010e51a48 pc=0x4a17de sync.runtime_SemacquireMutex(0xc010e51b70?, 0x20?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc010e51ae0 sp=0xc010e51ab0 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x7f271bcbe220?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc010e51b38 sp=0xc010e51ae0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc0225de2c0?, 0xc010e51bc8?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc010e51b80 sp=0xc010e51b38 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0225de2c0, {0x17?, 0x17?}, 0x0, 0xc00d3e1d78, 0xc00d9a1210) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc010e51c80 sp=0xc010e51b80 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc00d3e1b00, 0x60?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc010e51d20 sp=0xc010e51c80 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc00d3e1b00, {0xc006a8a260, 0x6, 0x1e}, 0x0?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc010e51dd0 sp=0xc010e51d20 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc0134b25a0?, 0x0?, 0x0?}, {0xc006a8a260?, 0x6?, 0xc00339e6e0?}, 0x30?, 0x29?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc010e51e38 sp=0xc010e51dd0 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc00d9a27e0?, {0xc0134b25a0?, 0x1?, 0x1e?}, {0xc006a8a260?, 0x10?, 0x1e?}, 0x38?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc010e51e88 sp=0xc010e51e38 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00d9a27e0, {0xc0134b25a0, 0x6, 0x50}, {0xc006a8a260, 0x6, 0x1e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc010e51ed8 sp=0xc010e51e88 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00d9a1420, {0xc006a8a260?, 0x0?, 0x724e61746a4b4863?}, 0x5?, 0x5?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc010e52008 sp=0xc010e51ed8 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00d9a1420, {0xc0134b25a0?, 0xa?, 0x6?}, {0xc006a8a260?, 0x6e4f4d7268595263?, 0x655a516a4f64555a?}, 0x53?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc010e52048 sp=0xc010e52008 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc00d9a1288, {0xc0134b25a0?, 0x50?, 0xc006a8a260?}, {0xc006a8a260, 0x6, 0x1e}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc010e520a0 sp=0xc010e52048 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00d9a0f00, {0xc006a8a260, 0x6, 0x1e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc010e52138 sp=0xc010e520a0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc003ca0028, {{0xc00fdc0518, 0x5, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc010e52188 sp=0xc010e52138 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc005bd9700, {{0xc00fdc0518, 0x5, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:489 +0x14a fp=0xc010e52248 sp=0xc010e52188 pc=0x180cdaa github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x62b4f18, 0xc005bd9700}, {{0xc00fdc0518, 0x5, 0x8}, {0x0, 0x0, 0x0}}, 0xc003a2e360) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1129 +0xc3 fp=0xc010e52698 sp=0xc010e52248 pc=0x1817843 github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x624a948, 0xc019ee5980}, {0x62b8438?, 0xc003ca0000?}, {0x62b4f18, 0xc005bd9700}, 0xc00262e000, {0xc00fdc0518, 0x5, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1650 +0x165 fp=0xc010e533a8 sp=0xc010e52698 pc=0x181aa25 github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x624a948, 0xc019ee5980}, {0x62b8438, 0xc003ca0000}, {0x62b4f18, 0xc005bd9700}, 0x9?, {0xc00fdc0518, 0x5, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1414 +0x1cd fp=0xc010e534b0 sp=0xc010e533a8 pc=0x18197cd github.com/cockroachdb/cockroach/pkg/storage.MVCCPut({0x624a948, 0xc019ee5980}, {0x7f275f2b7158?, 0xc003ca0000}, 0xc00262e000, {0xc00fdc0518, 0x5, 0x8}, {0x17600a76e06f4b91, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1333 +0x2a5 fp=0xc010e535c0 sp=0xc010e534b0 pc=0x1819005 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Put({_, _}, {_, _}, {{0x62cc4b0, 0xc013f614e0}, {{0x17600a76e06f4b91, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_put.go:59 +0x311 fp=0xc010e53bf0 sp=0xc010e535c0 pc=0x1cfa251 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e06f4b91, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc010e54480 sp=0xc010e53bf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc001c484b0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc010e562f0 sp=0xc010e54480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc004110a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc010e56940 sp=0xc010e562f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc004110a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc010e56e18 sp=0xc010e56940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76fe3cb091, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc010e57c98 sp=0xc010e56e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000100400?, {0x624a948, 0xc019ee5980}, {0xc00fdc0520, 0x8}, 0xc001c484b0, 0x8?, 0x0?, {{0x17600a76fe3cb091, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc010e58918 sp=0xc010e57c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc019ee5980}, {0xc00fdc0520, 0x8}, 0xc001c484b0, 0x400000004?, 0xc010e59968, {{0x17600a76fe3cb091, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc010e589d0 sp=0xc010e58918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc003861800, {0x624a948, 0xc019ee5980}, 0xc001c484b0, 0xc004110a00, 0xc010e59968, {{0x17600a76fe3cb091, 0x0, 0x0}, {0x17600a76e0c32cc3, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc010e59398 sp=0xc010e589d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc019ee5980}, 0xc001c484b0, 0xc004110a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc010e59b08 sp=0xc010e59398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc019ee5980}, 0xc001c484b0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc010e59e28 sp=0xc010e59b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e06f4b91, 0x0, 0x0}, 0x0, {0x17600a76e36bc23d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc010e5a1f0 sp=0xc010e59e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e06f4b91, 0x0, 0x0}, 0x0, {0x17600a76e36bc23d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc010e5ab78 sp=0xc010e5a1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xff3d15f6d74568b4?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e36bc23d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc010e5adc0 sp=0xc010e5ab78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc019ee5920}, {0x0?}, 0xc001c481e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc010e5b430 sp=0xc010e5adc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc019ee5860}, 0xc001c481e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc010e5b4d8 sp=0xc010e5b430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc019ee5860}, {0x52f0080?, 0xc001c481e0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc010e5b518 sp=0xc010e5b4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc019ee5860}, {0x52f0080, 0xc001c481e0}, 0xc00ed10740, 0xc00c072b88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc010e5b848 sp=0xc010e5b518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc019ee5860?}, {0x52f0080?, 0xc001c481e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc010e5b888 sp=0xc010e5b848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc019ee5860}, {0x52f0080, 0xc001c481e0}, 0xc010e5b920?, 0xc00d51fd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc010e5b8b8 sp=0xc010e5b888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc019ee5860?}, {0x52f0080?, 0xc001c481e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc010e5b8f8 sp=0xc010e5b8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc019ee5860?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc010e5b930 sp=0xc010e5b8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc019ee5860}, {0x203003?, 0x40?}, 0xc010e5b9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc010e5b9a8 sp=0xc010e5b930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc019ee5860?}, {0x52f0080?, 0xc001c481e0?}, 0x5003b00?, 0x7f278cfaaa01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc010e5ba20 sp=0xc010e5b9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc019ee5860?}, {0x52f0080?, 0xc001c481e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc010e5ba60 sp=0xc010e5ba20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc019ee5860}, {0x52f0080, 0xc001c481e0}, 0xc00ed10740, 0xc00c072b88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc010e5bab0 sp=0xc010e5ba60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc019ee5860}, 0xc0088aa180, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc010e5bb08 sp=0xc010e5bab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc01104a7e0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc010e5be48 sp=0xc010e5bb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc01104a7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc010e5bf68 sp=0xc010e5be48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc010e5bfe0 sp=0xc010e5bf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010e5bfe8 sp=0xc010e5bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498489 [select]: runtime.gopark(0xc010067208?, 0x2?, 0x38?, 0xad?, 0xc010067204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010067090 sp=0xc010067070 pc=0x4907d6 runtime.selectgo(0xc010067208, 0xc010067200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0100671d0 sp=0xc010067090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc058a58000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc010067238 sp=0xc0100671d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030826c60, {0x51b9a20?, 0xc0533f6900}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc010067340 sp=0xc010067238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc010067370 sp=0xc010067340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc02d889c20, 0xc010067448, 0xc010067418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0100673c8 sp=0xc010067370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc02d889c20, {0x51b9a20?, 0xc0533f6900?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc010067478 sp=0xc0100673c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00aeb2c00?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc017227950}, {0x51b9a20, 0xc0533f6900}, 0x0?, {0xc00a393c00, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0100674e0 sp=0xc010067478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb2c00}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227950}, {0x51b9a20, 0xc0533f6900}, 0x62d9a96d0dac56d1?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0100675d0 sp=0xc0100674e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00aeb2c00?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc017227950?}, {0x51b9a20?, 0xc0533f6900?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc010067670 sp=0xc0100675d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277490, {0x624a948, 0xc00aeb2c00}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0100676f0 sp=0xc010067670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277490?}}, {0x624a948, 0xc00aeb2c00}, 0xc017227950, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0100677c8 sp=0xc0100676f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00aeb2c00?}, 0x624a948?, {0x0?, 0xc005751d20?, 0xc00c88d928?}) :1 +0x65 fp=0xc010067818 sp=0xc0100677c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00d119c80, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010067ab8 sp=0xc010067818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00d119c80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e509a3f9, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010067bf0 sp=0xc010067ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e509a3f9, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010068338 sp=0xc010067bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e509a3f9, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010068c38 sp=0xc010068338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e509a3f9, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0100694a0 sp=0xc010068c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e509a3f9, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0100698c0 sp=0xc0100694a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0100699f8 sp=0xc0100698c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00564ddd0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc010069c00 sp=0xc0100699f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00564ddd0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc010069f30 sp=0xc010069c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010069fe0 sp=0xc010069f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010069fe8 sp=0xc010069fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498487 [select]: runtime.gopark(0xc00f89f208?, 0x2?, 0x30?, 0xac?, 0xc00f89f204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f89f090 sp=0xc00f89f070 pc=0x4907d6 runtime.selectgo(0xc00f89f208, 0xc00f89f200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f89f1d0 sp=0xc00f89f090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc02d889b00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00f89f238 sp=0xc00f89f1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030826bb0, {0x51b9a20?, 0xc0533f6880}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00f89f340 sp=0xc00f89f238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00f89f370 sp=0xc00f89f340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc02d8898c0, 0xc00f89f448, 0xc00f89f418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00f89f3c8 sp=0xc00f89f370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc02d8898c0, {0x51b9a20?, 0xc0533f6880?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00f89f478 sp=0xc00f89f3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00aeb2930?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc017227860}, {0x51b9a20, 0xc0533f6880}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00f89f4e0 sp=0xc00f89f478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb2930}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227860}, {0x51b9a20, 0xc0533f6880}, 0x22bcb4568c22ef29?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00f89f5d0 sp=0xc00f89f4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00aeb2930?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc017227860?}, {0x51b9a20?, 0xc0533f6880?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00f89f670 sp=0xc00f89f5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277470, {0x624a948, 0xc00aeb2930}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00f89f6f0 sp=0xc00f89f670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277470?}}, {0x624a948, 0xc00aeb2930}, 0xc017227860, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00f89f7c8 sp=0xc00f89f6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00aeb2930?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc00c88d928?}) :1 +0x65 fp=0xc00f89f818 sp=0xc00f89f7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc008389ce0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f89fab8 sp=0xc00f89f818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc008389ce0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e506d85a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f89fbf0 sp=0xc00f89fab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e506d85a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f8a0338 sp=0xc00f89fbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e506d85a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f8a0c38 sp=0xc00f8a0338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e506d85a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f8a14a0 sp=0xc00f8a0c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e506d85a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f8a18c0 sp=0xc00f8a14a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00f8a19f8 sp=0xc00f8a18c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00564da70}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00f8a1c00 sp=0xc00f8a19f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00564da70}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00f8a1f30 sp=0xc00f8a1c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f8a1fe0 sp=0xc00f8a1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f8a1fe8 sp=0xc00f8a1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 483587 [sleep]: runtime.gopark(0x4b88475aa59?, 0xc825f2?, 0x23?, 0x54?, 0x8055671682?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00da29df0 sp=0xc00da29dd0 pc=0x4907d6 time.Sleep(0x12d03d46e) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00da29e30 sp=0xc00da29df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc007852a50, 0xc825f2, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00da29e88 sp=0xc00da29e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc007852a50, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00da29ec8 sp=0xc00da29e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc006e25400?, 0xa, 0xc00c823f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00da29fa8 sp=0xc00da29ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00da29fe0 sp=0xc00da29fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00da29fe8 sp=0xc00da29fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498387 [select]: runtime.gopark(0xc00fd0d808?, 0x3?, 0x4?, 0x0?, 0xc00fd0d4d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fd0d258 sp=0xc00fd0d238 pc=0x4907d6 runtime.selectgo(0xc00fd0d808, 0xc00fd0d4cc, 0x17600a76c64d214c?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fd0d398 sp=0xc00fd0d258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc003893020}, 0xc005291860, 0xc00d1b4a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00fd0db08 sp=0xc00fd0d398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc003893020}, 0xc005291860, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00fd0de28 sp=0xc00fd0db08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76c4d26dc4, 0x0, 0x0}, 0x0, {0x17600a76e00c6dca, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00fd0e1f0 sp=0xc00fd0de28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76c4d26dc4, 0x0, 0x0}, 0x0, {0x17600a76e00c6dca, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00fd0eb78 sp=0xc00fd0e1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e00c6dca, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00fd0edc0 sp=0xc00fd0eb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc003892f60}, {0x0?}, 0xc005291770) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00fd0f430 sp=0xc00fd0edc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc003892e40}, 0xc005291770) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00fd0f4d8 sp=0xc00fd0f430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc003892e40}, {0x52f0080?, 0xc005291770}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00fd0f518 sp=0xc00fd0f4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc003892e40}, {0x52f0080, 0xc005291770}, 0xc012ae46e0, 0xc0174e0b10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00fd0f848 sp=0xc00fd0f518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc003892e40?}, {0x52f0080?, 0xc005291770?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00fd0f888 sp=0xc00fd0f848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc003892e40}, {0x52f0080, 0xc005291770}, 0xc00fd0f920?, 0xc008152fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00fd0f8b8 sp=0xc00fd0f888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc003892e40?}, {0x52f0080?, 0xc005291770?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fd0f8f8 sp=0xc00fd0f8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc003892e40?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00fd0f930 sp=0xc00fd0f8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc003892e40}, {0x7f278cfaa5b8?, 0x40?}, 0xc00fd0f9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00fd0f9a8 sp=0xc00fd0f930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc003892e40?}, {0x52f0080?, 0xc005291770?}, 0x5003b00?, 0x7f278cfaa501?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00fd0fa20 sp=0xc00fd0f9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc003892e40?}, {0x52f0080?, 0xc005291770?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fd0fa60 sp=0xc00fd0fa20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc003892e40}, {0x52f0080, 0xc005291770}, 0xc012ae46e0, 0xc0174e0b10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00fd0fab0 sp=0xc00fd0fa60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc003892e40}, 0xc0111619e0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00fd0fb08 sp=0xc00fd0fab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc00d643200, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00fd0fe48 sp=0xc00fd0fb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc00d643200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00fd0ff68 sp=0xc00fd0fe48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00fd0ffe0 sp=0xc00fd0ff68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fd0ffe8 sp=0xc00fd0ffe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 462794 [sleep]: runtime.gopark(0x4b89be6b975?, 0x400ea6?, 0xcc?, 0xed?, 0x807937b027?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00398ddf0 sp=0xc00398ddd0 pc=0x4907d6 time.Sleep(0x139636064) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00398de30 sp=0xc00398ddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc018c3e6f0, 0x400ea6, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00398de88 sp=0xc00398de30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc018c3e6f0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00398dec8 sp=0xc00398de88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc004d5f500?, 0x2d, 0xc0106cff70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00398dfa8 sp=0xc00398dec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00398dfe0 sp=0xc00398dfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00398dfe8 sp=0xc00398dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 476646 [sleep]: runtime.gopark(0x4b898378de0?, 0x8019a1?, 0x89?, 0xaa?, 0x8073466ce8?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc006aeddf0 sp=0xc006aeddd0 pc=0x4907d6 time.Sleep(0x13721488d) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc006aede30 sp=0xc006aeddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00c1c3290, 0x8019a1, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc006aede88 sp=0xc006aede30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00c1c3290, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc006aedec8 sp=0xc006aede88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc016e3c000?, 0xc, 0xc004a0efb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc006aedfa8 sp=0xc006aedec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc006aedfe0 sp=0xc006aedfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc006aedfe8 sp=0xc006aedfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 449041 [sleep]: runtime.gopark(0x4b92b7da6c2?, 0x400ea1?, 0xaa?, 0x18?, 0x81156d38f9?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d01bdf0 sp=0xc00d01bdd0 pc=0x4907d6 time.Sleep(0x14601fac9) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00d01be30 sp=0xc00d01bdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc006e41500, 0x400ea1, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00d01be88 sp=0xc00d01be30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc006e41500, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00d01bec8 sp=0xc00d01be88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00400ca00?, 0x19, 0xc004a12fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00d01bfa8 sp=0xc00d01bec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00d01bfe0 sp=0xc00d01bfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d01bfe8 sp=0xc00d01bfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498168 [select]: runtime.gopark(0xc007a382d8?, 0x2?, 0x8?, 0x24?, 0xc007a382d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007a38160 sp=0xc007a38140 pc=0x4907d6 runtime.selectgo(0xc007a382d8, 0xc007a382d0, 0x10?, 0x0, 0x201?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc007a382a0 sp=0xc007a38160 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00cd347e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc007a38308 sp=0xc007a382a0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc004e32d10, {0x51b9a20?, 0xc003b8ae80}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc007a38410 sp=0xc007a38308 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x1f8?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc007a38440 sp=0xc007a38410 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc00cd345a0, 0xc007a38518, 0xc007a384e8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc007a38498 sp=0xc007a38440 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00cd345a0, {0x51b9a20?, 0xc003b8ae80?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc007a38548 sp=0xc007a38498 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00cc2c6c0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00723a870}, {0x51b9a20, 0xc003b8ae80}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc007a385b0 sp=0xc007a38548 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00cc2c6c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00723a870}, {0x51b9a20, 0xc003b8ae80}, 0x47495bf30619cf11?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc007a386a0 sp=0xc007a385b0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x16d80553546696a7?, {0x624a948?, 0xc00cc2c6c0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00723a870?}, {0x51b9a20?, 0xc003b8ae80?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc007a38740 sp=0xc007a386a0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc011d2b578, {0x624a948, 0xc00cc2c6c0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc007a387c0 sp=0xc007a38740 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc011d2b578?}}, {0x624a948, 0xc00cc2c6c0}, 0xc00723a870, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc007a38898 sp=0xc007a387c0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xa354900?, {0x624a948?, 0xc00cc2c6c0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00a3549f8?}) :1 +0x65 fp=0xc007a388e8 sp=0xc007a38898 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01546b740, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc007a38b88 sp=0xc007a388e8 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01546b740, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d78fca11, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc007a38cc0 sp=0xc007a38b88 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d78fca11, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc007a39408 sp=0xc007a38cc0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d78fca11, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc007a39d08 sp=0xc007a39408 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x624a948, 0xc00cc2c6c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1506 +0x1bc fp=0xc007a39f30 sp=0xc007a39d08 pc=0x19ca1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc007a39fe0 sp=0xc007a39f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007a39fe8 sp=0xc007a39fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 476821 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1df?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c0e6f58 sp=0xc00c0e6f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7360?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00c0e6f90 sp=0xc00c0e6f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7278, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00c0e6fb0 sp=0xc00c0e6f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc002da7a00?, 0xc005a02000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00c0e6fd8 sp=0xc00c0e6fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc002da7a00, {0xc005a02000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00c0e7058 sp=0xc00c0e6fd8 pc=0x535efa net.(*netFD).Read(0xc002da7a00, {0xc005a02000?, 0x7b5a68?, 0xc002da7a00?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00c0e70a0 sp=0xc00c0e7058 pc=0x7a20c9 net.(*conn).Read(0xc00d480410, {0xc005a02000?, 0x164290f?, 0xc00d480410?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00c0e70e8 sp=0xc00c0e70a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0000c8300, {0xc005a02000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00c0e7130 sp=0xc00c0e70e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc006db0408, {0xc005a02000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00c0e71d0 sp=0xc00c0e7130 pc=0x370f896 bufio.(*Reader).fill(0xc00391b090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00c0e7208 sp=0xc00c0e71d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00391b090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00c0e7228 sp=0xc00c0e7208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00391b668?, {0x62295c8, 0xc00391b090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00c0e7260 sp=0xc00c0e7228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00391b000, 0xc00faa24d0, 0xc00c0e7442, 0xc00c0e7444, {0x623b718, 0xc004fced80}, 0xc0059fa000, 0xc0123f43c0, 0xc00c0e7440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00c0e7398 sp=0xc00c0e7260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00391b000, {0x624a948, 0xc01551e3f0}, 0xc00faa24c0, 0xc0059fa000, 0xc0171f1f20, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00c0e7768 sp=0xc00c0e7398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc01551e3f0}, {0x628a538, 0xc0000c8300}, {{{0xc0013fa009, 0x4}}, 0x1, 0xc0171f1f50, 0xc0171f1f80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00c0e78a8 sp=0xc00c0e7768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc01551e3f0}, {0x628a538?, 0xc0000c8300}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00c0e7e20 sp=0xc00c0e78a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0171f1e30?}, {0x628a538, 0xc0000c8300}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00c0e7eb0 sp=0xc00c0e7e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0171f1e30}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00c0e7f30 sp=0xc00c0e7eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c0e7fe0 sp=0xc00c0e7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c0e7fe8 sp=0xc00c0e7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498318 [select]: runtime.gopark(0xc00a629c18?, 0x3?, 0xa0?, 0xe?, 0xc00a629b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a6298f0 sp=0xc00a6298d0 pc=0x4907d6 runtime.selectgo(0xc00a629c18, 0xc00a629b34, 0xc008152240?, 0x0, 0x19?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a629a30 sp=0xc00a6298f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc0034e94d0}, {0x6221560, 0xc0051cc6c0}, {0x46?, {0xc00a629e01?, 0xa629e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00a629da0 sp=0xc00a629a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc0034e94d0?}, 0x164, 0xc0051cc6c0, 0x1a?, 0xc0120e6a50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00a629e70 sp=0xc00a629da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc0034e94d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00a629f30 sp=0xc00a629e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a629fe0 sp=0xc00a629f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a629fe8 sp=0xc00a629fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 11655 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0xca3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cc6ef58 sp=0xc00cc6ef38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f2435b0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00cc6ef90 sp=0xc00cc6ef58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f2434c8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00cc6efb0 sp=0xc00cc6ef90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00418fc00?, 0xc00db24000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00cc6efd8 sp=0xc00cc6efb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00418fc00, {0xc00db24000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00cc6f058 sp=0xc00cc6efd8 pc=0x535efa net.(*netFD).Read(0xc00418fc00, {0xc00db24000?, 0x7b5a68?, 0xc00418fc00?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00cc6f0a0 sp=0xc00cc6f058 pc=0x7a20c9 net.(*conn).Read(0xc005b51458, {0xc00db24000?, 0x164290f?, 0xc005b51458?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00cc6f0e8 sp=0xc00cc6f0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00d173020, {0xc00db24000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00cc6f130 sp=0xc00cc6f0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0138f7fe0, {0xc00db24000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00cc6f1d0 sp=0xc00cc6f130 pc=0x370f896 bufio.(*Reader).fill(0xc00d8c2c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00cc6f208 sp=0xc00cc6f1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00d8c2c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00cc6f228 sp=0xc00cc6f208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00d8c3268?, {0x62295c8, 0xc00d8c2c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00cc6f260 sp=0xc00cc6f228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00d8c2c00, 0xc000998550, 0xc00cc6f442, 0xc00cc6f444, {0x623b718, 0xc00b6a97a0}, 0xc0059fa000, 0xc001a67cd4, 0xc00cc6f440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00cc6f398 sp=0xc00cc6f260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00d8c2c00, {0x624a948, 0xc0034ad230}, 0xc000998540, 0xc0059fa000, 0xc0034ad0e0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00cc6f768 sp=0xc00cc6f398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0034ad230}, {0x628a538, 0xc00d173020}, {{{0xc00db1e055, 0x4}}, 0x1, 0xc0034ad110, 0xc0034ad140, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00cc6f8a8 sp=0xc00cc6f768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0034ad230}, {0x628a538?, 0xc00d173020}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00cc6fe20 sp=0xc00cc6f8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0034ad020?}, {0x628a538, 0xc00d173020}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00cc6feb0 sp=0xc00cc6fe20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0034ad020}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00cc6ff30 sp=0xc00cc6feb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cc6ffe0 sp=0xc00cc6ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cc6ffe8 sp=0xc00cc6ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 117853 [select, 6 minutes]: runtime.gopark(0xc002c93f98?, 0x2?, 0x0?, 0x0?, 0xc002c93f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002c93e00 sp=0xc002c93de0 pc=0x4907d6 runtime.selectgo(0xc002c93f98, 0xc002c93f78, 0xc01266ad20?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc002c93f40 sp=0xc002c93e00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00a393c40) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc002c93fc8 sp=0xc002c93f40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc002c93fe0 sp=0xc002c93fc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002c93fe8 sp=0xc002c93fe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 498367 [select]: runtime.gopark(0xc00cf53c18?, 0x3?, 0xd2?, 0x39?, 0xc00cf53b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cf538f0 sp=0xc00cf538d0 pc=0x4907d6 runtime.selectgo(0xc00cf53c18, 0xc00cf53b34, 0xc00a1d6680?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00cf53a30 sp=0xc00cf538f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc012b412c0}, {0x6221560, 0xc032e02fc0}, {0x46?, {0xc00cf53e01?, 0xcf53e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00cf53da0 sp=0xc00cf53a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc012b412c0?}, 0x164, 0xc032e02fc0, 0x1?, 0xc00d57ecc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00cf53e70 sp=0xc00cf53da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc012b412c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00cf53f30 sp=0xc00cf53e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00cf53fe0 sp=0xc00cf53f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cf53fe8 sp=0xc00cf53fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498222 [chan receive]: runtime.gopark(0xc014ba6e08?, 0x0?, 0x0?, 0x0?, 0xc00a443f08?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a443ed8 sp=0xc00a443eb8 pc=0x4907d6 runtime.chanrecv(0xc002b1f680, 0xc00a443fb0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc00a443f68 sp=0xc00a443ed8 pc=0x45a91b runtime.chanrecv2(0x1054d60?, 0xc002a40360?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc00a443f90 sp=0xc00a443f68 pc=0x45a458 github.com/cockroachdb/pebble/sstable.(*writeQueue).runWorker(0xc006f46640) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:105 +0x4c fp=0xc00a443fc8 sp=0xc00a443f90 pc=0x10d7d0c github.com/cockroachdb/pebble/sstable.newWriteQueue.func1() github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0x26 fp=0xc00a443fe0 sp=0xc00a443fc8 pc=0x10d7606 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a443fe8 sp=0xc00a443fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/sstable.newWriteQueue github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0xd6 goroutine 115742 [select]: runtime.gopark(0xc0036bce18?, 0x2?, 0x50?, 0x4a?, 0xc0036bcdfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0036bcc68 sp=0xc0036bcc48 pc=0x4907d6 runtime.selectgo(0xc0036bce18, 0xc0036bcdf8, 0xc003f56800?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0036bcda8 sp=0xc0036bcc68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc006258000, {0x624a8a0, 0xc00c7c4640}, 0xc002e4b4a0?, 0xc003f8e8a0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc0036bcf20 sp=0xc0036bcda8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc00c7c4640?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc0036bcf58 sp=0xc0036bcf20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc0036bcf78 sp=0xc0036bcf58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc0036bcfe0 sp=0xc0036bcf78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0036bcfe8 sp=0xc0036bcfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498462 [runnable]: runtime.gopark(0xae1b64601d4?, 0x4179756161677063?, 0x75?, 0x71?, 0x6b6368526147636a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010bcd6d8 sp=0xc010bcd6b8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x59?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc010bcd740 sp=0xc010bcd6d8 pc=0x4a17de sync.runtime_SemacquireMutex(0x78726b635a6f445a?, 0x5a?, 0x6e6e705269507a4a?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc010bcd770 sp=0xc010bcd740 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc010bcd7c8 sp=0xc010bcd770 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x1801f9e?, 0x7f270c2d5aa0?, 0x3cc5?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc010bcd810 sp=0xc010bcd7c8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00858db80, {0x8?, 0x11?}, 0x0, 0x0, 0xc00008f710) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc010bcd910 sp=0xc010bcd810 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2727 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc002120000, {0xc007fca440, 0x21, 0x40}, {0xc00a4e9f20, 0x21, 0x21}, 0x60?, 0x61?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:847 +0xf8 fp=0xc010bcd978 sp=0xc010bcd910 pc=0x10c5078 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc000090840?, {0xc007fca440?, 0x1?, 0x21?}, {0xc00a4e9f20?, 0x10?, 0x21?}, 0x4a?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc010bcd9c8 sp=0xc010bcd978 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc000090840, {0xc007fca440, 0x21, 0x40}, {0xc00a4e9f20, 0x21, 0x21}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc010bcda18 sp=0xc010bcd9c8 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00008f920, {0xc00a4e9f20?, 0x4c6772694a6a7478?, 0x100586d726d73?}, 0x20?, 0x20?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc010bcdb48 sp=0xc010bcda18 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00008f920, {0xc007fca440?, 0x0?, 0x21?}, {0xc00a4e9f20?, 0x4a59d2?, 0x21?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc010bcdb88 sp=0xc010bcdb48 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc00008f788, {0xc007fca440?, 0x40?, 0xc00a4e9f20?}, {0xc00a4e9f20, 0x21, 0x21}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc010bcdbe0 sp=0xc010bcdb88 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00008f400, {0xc00a4e9f20, 0x21, 0x21}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc010bcdc78 sp=0xc010bcdbe0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0053a4028, {{0xc00a4e9ef0, 0x20, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc010bcdcc8 sp=0xc010bcdc78 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc005047200, {0x624a948, 0xc03064b860}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc010bcdd30 sp=0xc010bcdcc8 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc03064b860}, {0x62b4fe0, 0xc0053a4028}, {0xc00a4e9ef0, 0x20, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc010bce300 sp=0xc010bcdd30 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc03064b860}, {0x628f6b0?, 0xc0053a4000?}, {0xc00a4e9ef0, 0x20, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc010bce480 sp=0xc010bce300 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc03064b860?}, {0x628f6b0?, 0xc0053a4000?}, {0xc00a4e9ef0?, 0x31?, 0x14?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc010bce570 sp=0xc010bce480 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.EndTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc011928300}, {{0x17600a76db3ed40a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:238 +0x2fa fp=0xc010bcfbf0 sp=0xc010bce570 pc=0x1ce7a1a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76db3ed40a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc010bd0480 sp=0xc010bcfbf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0120d2a50, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc010bd22f0 sp=0xc010bd0480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc004ef7860, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc010bd2940 sp=0xc010bd22f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc004ef7860, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc010bd2e18 sp=0xc010bd2940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76f90c390a, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc010bd3c98 sp=0xc010bd2e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000100400?, {0x624a948, 0xc03064b860}, {0xc00413b188, 0x8}, 0xc0120d2a50, 0x8?, 0x0?, {{0x17600a76f90c390a, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc010bd4918 sp=0xc010bd3c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc03064b860}, {0xc00413b188, 0x8}, 0xc0120d2a50, 0x400000004?, 0xc010bd5968, {{0x17600a76f90c390a, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc010bd49d0 sp=0xc010bd4918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc002228c00, {0x624a948, 0xc03064b860}, 0xc0120d2a50, 0xc004ef7860, 0xc010bd5968, {{0x17600a76f90c390a, 0x0, 0x0}, {0x17600a76dc1b4da0, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc010bd5398 sp=0xc010bd49d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc002228c00, {0x624a948, 0xc03064b860}, 0xc0120d2a50, 0xc004ef7860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc010bd5b08 sp=0xc010bd5398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc002228c00, {0x624a948, 0xc03064b860}, 0xc0120d2a50, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc010bd5e28 sp=0xc010bd5b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc002228c00, {_, _}, {{{0x17600a76db3ed40a, 0x0, 0x0}, 0x0, {0x17600a76e20de0aa, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc010bd61f0 sp=0xc010bd5e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76db3ed40a, 0x0, 0x0}, 0x0, {0x17600a76e20de0aa, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc010bd6b78 sp=0xc010bd61f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc001b923c0?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e20de0aa, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc010bd6dc0 sp=0xc010bd6b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc03064b800}, {0x459c3d?}, 0xc0120d2960) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc010bd7430 sp=0xc010bd6dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc03064b6e0}, 0xc0120d2960) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc010bd74d8 sp=0xc010bd7430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc03064b6e0}, {0x52f0080?, 0xc0120d2960}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc010bd7518 sp=0xc010bd74d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc03064b6e0}, {0x52f0080, 0xc0120d2960}, 0xc002699360, 0xc03d7cb038) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc010bd7848 sp=0xc010bd7518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03064b6e0?}, {0x52f0080?, 0xc0120d2960?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc010bd7888 sp=0xc010bd7848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc03064b6e0}, {0x52f0080, 0xc0120d2960}, 0xc010bd7920?, 0xc00d1e7a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc010bd78b8 sp=0xc010bd7888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03064b6e0?}, {0x52f0080?, 0xc0120d2960?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc010bd78f8 sp=0xc010bd78b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc03064b6e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc010bd7930 sp=0xc010bd78f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc03064b6e0}, {0x203000?, 0x40?}, 0xc010bd79e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc010bd79a8 sp=0xc010bd7930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc03064b6e0?}, {0x52f0080?, 0xc0120d2960?}, 0x5003b00?, 0x7f278cfaaa01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc010bd7a20 sp=0xc010bd79a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03064b6e0?}, {0x52f0080?, 0xc0120d2960?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc010bd7a60 sp=0xc010bd7a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc03064b6e0}, {0x52f0080, 0xc0120d2960}, 0xc002699360, 0xc03d7cb038) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc010bd7ab0 sp=0xc010bd7a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc03064b6e0}, 0xc00823b860, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc010bd7b08 sp=0xc010bd7ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc013e857a0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc010bd7e48 sp=0xc010bd7b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc013e857a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc010bd7f68 sp=0xc010bd7e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc010bd7fe0 sp=0xc010bd7f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010bd7fe8 sp=0xc010bd7fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498459 [select]: runtime.gopark(0xc01081f208?, 0x2?, 0xb0?, 0xa7?, 0xc01081f204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01081f090 sp=0xc01081f070 pc=0x4907d6 runtime.selectgo(0xc01081f208, 0xc01081f200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01081f1d0 sp=0xc01081f090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc011e318c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01081f238 sp=0xc01081f1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018ac6e0, {0x51b9a20?, 0xc00d502280}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01081f340 sp=0xc01081f238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01081f370 sp=0xc01081f340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011e31440, 0xc01081f448, 0xc01081f418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01081f3c8 sp=0xc01081f370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011e31440, {0x51b9a20?, 0xc00d502280?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01081f478 sp=0xc01081f3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc011c54bd0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0081db860}, {0x51b9a20, 0xc00d502280}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01081f4e0 sp=0xc01081f478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011c54bd0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081db860}, {0x51b9a20, 0xc00d502280}, 0x52bfe6cf0933ad60?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01081f5d0 sp=0xc01081f4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc011c54bd0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0081db860?}, {0x51b9a20?, 0xc00d502280?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01081f670 sp=0xc01081f5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc012277328, {0x624a948, 0xc011c54bd0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01081f6f0 sp=0xc01081f670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc012277328?}}, {0x624a948, 0xc011c54bd0}, 0xc0081db860, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01081f7c8 sp=0xc01081f6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xba83830?, {0x624a948?, 0xc011c54bd0?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc00ba83928?}) :1 +0x65 fp=0xc01081f818 sp=0xc01081f7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00c924240, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01081fab8 sp=0xc01081f818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00c924240, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2938632, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01081fbf0 sp=0xc01081fab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2938632, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010820338 sp=0xc01081fbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2938632, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010820c38 sp=0xc010820338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2938632, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0108214a0 sp=0xc010820c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2938632, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0108218c0 sp=0xc0108214a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0108219f8 sp=0xc0108218c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc0032104b0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc010821c00 sp=0xc0108219f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc0032104b0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc010821f30 sp=0xc010821c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010821fe0 sp=0xc010821f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010821fe8 sp=0xc010821fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 120461 [select]: runtime.gopark(0xc0015b3f00?, 0x2?, 0x88?, 0x3d?, 0xc0015b3ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0015b3d60 sp=0xc0015b3d40 pc=0x4907d6 runtime.selectgo(0xc0015b3f00, 0xc0015b3ed0, 0x0?, 0x0, 0xc0015b3ef0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0015b3ea0 sp=0xc0015b3d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00a609f90, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc0015b3f30 sp=0xc0015b3ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0155fd440) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc0015b3f88 sp=0xc0015b3f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xce fp=0xc0015b3fe0 sp=0xc0015b3f88 pc=0xbedbae runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0015b3fe8 sp=0xc0015b3fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x19f3 goroutine 117879 [runnable]: runtime.gopark(0xc001509d80?, 0xb?, 0x0?, 0x0?, 0x144?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001509a80 sp=0xc001509a60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x43?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc001509ab8 sp=0xc001509a80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764538dc8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc001509ad8 sp=0xc001509ab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00f3c1580?, 0xc00fa74000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001509b00 sp=0xc001509ad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00f3c1580, {0xc00fa74000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc001509b80 sp=0xc001509b00 pc=0x535efa net.(*netFD).Read(0xc00f3c1580, {0xc00fa74000?, 0x50c7cc0?, 0xc00f8fef01?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc001509bc8 sp=0xc001509b80 pc=0x7a20c9 net.(*conn).Read(0xc00ff52ce0, {0xc00fa74000?, 0xc00fa74000?, 0xc011b13290?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc001509c10 sp=0xc001509bc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc00f30daa0, {0xc005876120, 0x9, 0xc0040c6d44?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc001509c48 sp=0xc001509c10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc00f30daa0}, {0xc005876120, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc001509c90 sp=0xc001509c48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc005876120?, 0x9?, 0xc011b13200?}, {0x62052c0?, 0xc00f30daa0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc001509ce0 sp=0xc001509c90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0058760e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc001509d90 sp=0xc001509ce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00f8fef00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc001509fc8 sp=0xc001509d90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc001509fe0 sp=0xc001509fc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001509fe8 sp=0xc001509fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 498372 [runnable]: runtime.cgocall(0x45a3130, 0xc002e0d978) GOROOT/src/runtime/cgocall.go:158 +0x5c fp=0xc002e0d950 sp=0xc002e0d918 pc=0x45789c github.com/cockroachdb/pebble/internal/manual._Cfunc_free(0x7f26b69ee440) github.com/cockroachdb/pebble/internal/manual/_cgo_gotypes.go:63 +0x45 fp=0xc002e0d978 sp=0xc002e0d950 pc=0x10a3205 github.com/cockroachdb/pebble/internal/manual.Free.func1(0x10a669b?) com_github_cockroachdb_cockroach/external/com_github_cockroachdb_pebble/internal/manual/manual.go:58 +0x3a fp=0xc002e0d9b0 sp=0xc002e0d978 pc=0x10a33da github.com/cockroachdb/pebble/internal/manual.Free({0x7f26b69ee440?, 0x4ccde5?, 0x8fc5240?}) com_github_cockroachdb_cockroach/external/com_github_cockroachdb_pebble/internal/manual/manual.go:58 +0x32 fp=0xc002e0d9d0 sp=0xc002e0d9b0 pc=0x10a3352 github.com/cockroachdb/pebble/internal/cache.(*Value).free(0xc002e0da30?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/value_normal.go:56 +0x57 fp=0xc002e0d9f8 sp=0xc002e0d9d0 pc=0x10a73b7 github.com/cockroachdb/pebble/internal/cache.(*Value).release(0x7f26b69ee440) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/value.go:44 +0x35 fp=0xc002e0da10 sp=0xc002e0d9f8 pc=0x10a7275 github.com/cockroachdb/pebble/internal/cache.(*entry).setValue(0x7f2712b00960?, 0xc002e0da50?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/entry.go:141 +0x5e fp=0xc002e0da28 sp=0xc002e0da10 pc=0x10a625e github.com/cockroachdb/pebble/internal/cache.(*shard).metaDel(0xc0005ee8b0, 0x7f270f4339c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:367 +0x2f fp=0xc002e0da60 sp=0xc002e0da28 pc=0x10a486f github.com/cockroachdb/pebble/internal/cache.(*shard).metaEvict(0xc0005ee7e0?, 0x7f270f4339c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:470 +0x8a fp=0xc002e0da80 sp=0xc002e0da60 pc=0x10a4b8a github.com/cockroachdb/pebble/internal/cache.(*shard).EvictFile(0xc0005ee8b0, 0xc002941970?, 0xc00461fe00?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:260 +0xcf fp=0xc002e0daf8 sp=0xc002e0da80 pc=0x10a418f github.com/cockroachdb/pebble/internal/cache.(*Cache).EvictFile(...) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:764 github.com/cockroachdb/pebble.(*tableCacheShard).evict(0xc002941970, 0x1f88b, 0xc000932208, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:786 +0x2b4 fp=0xc002e0dba0 sp=0xc002e0daf8 pc=0x1168814 github.com/cockroachdb/pebble.(*tableCacheContainer).evict(0x4fabb80?, 0xc011492d20?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:152 +0x45 fp=0xc002e0dbd0 sp=0xc002e0dba0 pc=0x1165925 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles(0xc002a2cd80, 0x132b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3096 +0x9bb fp=0xc002e0ded0 sp=0xc002e0dbd0 pc=0x110bd3b github.com/cockroachdb/pebble.(*DB).maybeScheduleObsoleteTableDeletion.func1.1({0x624a948, 0xc015bf96b0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3158 +0xa5 fp=0xc002e0df20 sp=0xc002e0ded0 pc=0x110c865 runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc000084480?, 0xc015bf94a0?, 0xc002e0df80?}}, 0xc002e0dfc0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc002e0df90 sp=0xc002e0df20 pc=0xfd3223 github.com/cockroachdb/pebble.(*DB).maybeScheduleObsoleteTableDeletion.func1() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3152 +0x5c fp=0xc002e0dfe0 sp=0xc002e0df90 pc=0x110c79c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002e0dfe8 sp=0xc002e0dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).maybeScheduleObsoleteTableDeletion github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3151 +0xea goroutine 496386 [select]: runtime.gopark(0xc00b4d1300?, 0x3?, 0x4?, 0x0?, 0xc00b4d0fca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b4d0d50 sp=0xc00b4d0d30 pc=0x4907d6 runtime.selectgo(0xc00b4d1300, 0xc00b4d0fc4, 0x17600a75f2843af6?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b4d0e90 sp=0xc00b4d0d50 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc002ee41b0}, 0xc0081dbd10, 0xc0128fea00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00b4d1600 sp=0xc00b4d0e90 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc002ee41b0}, 0xc0081dbd10, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00b4d1920 sp=0xc00b4d1600 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a75f2843af6, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00b4d1ce8 sp=0xc00b4d1920 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a75f2843af6, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00b4d2670 sp=0xc00b4d1ce8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00b4d28b8 sp=0xc00b4d2670 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc002ee4150}, {0x1f0?}, 0xc0081dbb30) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00b4d2f28 sp=0xc00b4d28b8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc002ee40c0}, 0xc0081dbb30) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00b4d2fd0 sp=0xc00b4d2f28 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc002ee40c0?}, {0x52f0080?, 0xc0081dbb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00b4d3000 sp=0xc00b4d2fd0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc002ee40c0}, {0x52f0080, 0xc0081dbb30}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00b4d3330 sp=0xc00b4d3000 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc002ee40c0?}, {0x52f0080?, 0xc0081dbb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00b4d3370 sp=0xc00b4d3330 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc002ee40c0}, {0x52f0080, 0xc0081dbb30}, 0xc00b4d3408?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00b4d33a0 sp=0xc00b4d3370 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc002ee40c0?}, {0x52f0080?, 0xc0081dbb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00b4d33e0 sp=0xc00b4d33a0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc002ee40c0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00b4d3418 sp=0xc00b4d33e0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc002ee40c0}, {0xc44d59?, 0xc010711500?}, 0xc00b4d34d0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00b4d3490 sp=0xc00b4d3418 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc002ee40c0?}, {0x52f0080?, 0xc0081dbb30?}, 0x51b9a20?, 0x0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00b4d3508 sp=0xc00b4d3490 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc002ee40c0?}, {0x52f0080?, 0xc0081dbb30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00b4d3548 sp=0xc00b4d3508 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc002ee40c0?}, {0x477f27?, 0x4?}, {0x52f0080?, 0xc0081dbb30?}, {0x51b9a20?, 0xc00d502580}, 0x203000?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00b4d3588 sp=0xc00b4d3548 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc002ee40c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081dbb30}, {0x51b9a20, 0xc00d502580}, 0xc00b4d3700?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00b4d3678 sp=0xc00b4d3588 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc002ee40c0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081dbb30}, {0x51b9a20, 0xc00d502580}, 0xc00b4d37e0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00b4d3710 sp=0xc00b4d3678 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc011c55cb0}, 0xc0081dbb30, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00b4d3790 sp=0xc00b4d3710 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x6231bd8?, {0x624a948?, 0xc011c55cb0?}, 0x624a948?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00b4d3818 sp=0xc00b4d3790 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc005c10000, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00b4d3ab8 sp=0xc00b4d3818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc005c10000, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00b4d3bf0 sp=0xc00b4d3ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00b4d4338 sp=0xc00b4d3bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00b4d4c38 sp=0xc00b4d4338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00b4d54a0 sp=0xc00b4d4c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed9e66, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00b4d58c0 sp=0xc00b4d54a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc010c82bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00b4d5b18 sp=0xc00b4d58c0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLocked(0xc010c82850, {0x624a948, 0xc012891410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:448 +0x388 fp=0xc00b4d5da8 sp=0xc00b4d5b18 pc=0x19e7f88 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc010c82850, {0x624a948, 0xc012891410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:415 +0xe5 fp=0xc00b4d5df8 sp=0xc00b4d5da8 pc=0x19e7b25 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc010c82850, {0x624a948, 0xc012891410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:385 +0x1b1 fp=0xc00b4d5eb0 sp=0xc00b4d5df8 pc=0x19e7831 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc012891410?}) :1 +0x39 fp=0xc00b4d5ed8 sp=0xc00b4d5eb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc012891410}, {0x53ddbcd?, 0x100000001?}, 0xc00a96bfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00b4d5f50 sp=0xc00b4d5ed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00b4d5fe0 sp=0xc00b4d5f50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b4d5fe8 sp=0xc00b4d5fe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 496480 [select]: runtime.gopark(0xc00acbfe60?, 0x3?, 0x0?, 0x0?, 0xc00acbfe32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00acbfcb8 sp=0xc00acbfc98 pc=0x4907d6 runtime.selectgo(0xc00acbfe60, 0xc00acbfe2c, 0xc00794d5f0?, 0x0, 0xc00a48fe40?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00acbfdf8 sp=0xc00acbfcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc00f6017d0, {0x624a948, 0xc00794d5f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00acbfeb0 sp=0xc00acbfdf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc00794d5f0?}) :1 +0x39 fp=0xc00acbfed8 sp=0xc00acbfeb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc00794d5f0}, {0x53ddbcd?, 0x1054e20?}, 0xc00a48ffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00acbff50 sp=0xc00acbfed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00acbffe0 sp=0xc00acbff50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00acbffe8 sp=0xc00acbffe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 455089 [sleep]: runtime.gopark(0x4b853c8b143?, 0x340c7e?, 0xef?, 0x41?, 0x801a913a56?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00da24df0 sp=0xc00da24dd0 pc=0x4907d6 time.Sleep(0x122daf4f4) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00da24e30 sp=0xc00da24df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc044c7fc50, 0x340c7e, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00da24e88 sp=0xc00da24e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc044c7fc50, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00da24ec8 sp=0xc00da24e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00467ca00?, 0x18, 0xc00c24bfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00da24fa8 sp=0xc00da24ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00da24fe0 sp=0xc00da24fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00da24fe8 sp=0xc00da24fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 87450 [semacquire, 6 minutes]: runtime.gopark(0xc00d243a68?, 0x46159f?, 0x80?, 0xa4?, 0xc004f654d0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002f499f0 sp=0xc002f499d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc00c7c4690, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc002f49a58 sp=0xc002f499f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc002f49a88 sp=0xc002f49a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc00d243ae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc002f49ab0 sp=0xc002f49a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc00c7c4680) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc002f49ac8 sp=0xc002f49ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00c7c4680?, {0x624a8a0?, 0xc00c7c4640?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc002f49af8 sp=0xc002f49ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc0196d6000?}, {0xc00d243c68, 0x2, 0xc0196d6000?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc002f49b48 sp=0xc002f49af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc006258000, {0x624a948, 0xc0196d6000}, 0xc003f56800) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc002f49e68 sp=0xc002f49b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc002e4b4a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc002f49f30 sp=0xc002f49e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc002f49fe0 sp=0xc002f49f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002f49fe8 sp=0xc002f49fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 401799 [sleep]: runtime.gopark(0x4b853476f38?, 0x1012201?, 0xc2?, 0x76?, 0x80190d6f22?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a4bdf0 sp=0xc005a4bdd0 pc=0x4907d6 time.Sleep(0x121d868ea) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc005a4be30 sp=0xc005a4bdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc015afb320, 0x1012201, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc005a4be88 sp=0xc005a4be30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc015afb320, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc005a4bec8 sp=0xc005a4be88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc002c8ff80?, {0xc004211900?, 0x1d, 0xc002c8ff70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc005a4bfa8 sp=0xc005a4bec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc005a4bfe0 sp=0xc005a4bfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a4bfe8 sp=0xc005a4bfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 344534 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc010143640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0101435d0 sp=0xc0101435b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00a453510, 0x473) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc010143618 sp=0xc0101435d0 pc=0x4be30c sync.(*Cond).Wait(0xc0118e7000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc010143650 sp=0xc010143618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0118e7668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc010143708 sp=0xc010143650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0040e0c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc010143cb8 sp=0xc010143708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0040e0c00, {0x624a8a0, 0xc00a453540}, 0x41178b8000000000?, 0xc00cf55dc0?, 0xc000a14820?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc010143d70 sp=0xc010143cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00a453540?}, {0xc0059fa000?}, 0x3?, 0xc001c10720?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc010143e20 sp=0xc010143d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc010143fe0 sp=0xc010143e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010143fe8 sp=0xc010143fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498490 [select]: runtime.gopark(0xc010c25208?, 0x2?, 0x40?, 0xae?, 0xc010c25204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010c25090 sp=0xc010c25070 pc=0x4907d6 runtime.selectgo(0xc010c25208, 0xc010c25200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc010c251d0 sp=0xc010c25090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc058a58480) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc010c25238 sp=0xc010c251d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030826fd0, {0x51b9a20?, 0xc0533f6980}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc010c25340 sp=0xc010c25238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc010c25370 sp=0xc010c25340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc058a58120, 0xc010c25448, 0xc010c25418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc010c253c8 sp=0xc010c25370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc058a58120, {0x51b9a20?, 0xc0533f6980?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc010c25478 sp=0xc010c253c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00aeb2f90?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc017227a40}, {0x51b9a20, 0xc0533f6980}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc010c254e0 sp=0xc010c25478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb2f90}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227a40}, {0x51b9a20, 0xc0533f6980}, 0x518f6a080895b5e2?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc010c255d0 sp=0xc010c254e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00aeb2f90?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc017227a40?}, {0x51b9a20?, 0xc0533f6980?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc010c25670 sp=0xc010c255d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0432774b0, {0x624a948, 0xc00aeb2f90}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc010c256f0 sp=0xc010c25670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0432774b0?}}, {0x624a948, 0xc00aeb2f90}, 0xc017227a40, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc010c257c8 sp=0xc010c256f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00aeb2f90?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc010c25818 sp=0xc010c257c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00f1f69c0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc010c25ab8 sp=0xc010c25818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00f1f69c0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50b3212, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc010c25bf0 sp=0xc010c25ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50b3212, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc010c26338 sp=0xc010c25bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50b3212, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc010c26c38 sp=0xc010c26338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50b3212, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc010c274a0 sp=0xc010c26c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50b3212, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc010c278c0 sp=0xc010c274a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc010c279f8 sp=0xc010c278c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00564de30}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc010c27c00 sp=0xc010c279f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00564de30}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc010c27f30 sp=0xc010c27c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010c27fe0 sp=0xc010c27f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010c27fe8 sp=0xc010c27fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49084 [select, 7 minutes]: runtime.gopark(0xc007b29fb0?, 0x2?, 0xa0?, 0x6f?, 0xc007b29f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dff6e18 sp=0xc00dff6df8 pc=0x4907d6 runtime.selectgo(0xc00dff6fb0, 0xc007b29f90, 0xc00296b6c0?, 0x0, 0xc004caa680?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dff6f58 sp=0xc00dff6e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc00dff6fe0 sp=0xc00dff6f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dff6fe8 sp=0xc00dff6fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 465034 [sleep]: runtime.gopark(0x4b90abc95fb?, 0x8019a9?, 0xcd?, 0xd6?, 0x80ef4c2d21?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e7ebdf0 sp=0xc00e7ebdd0 pc=0x4907d6 time.Sleep(0x140a1ff07) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00e7ebe30 sp=0xc00e7ebdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0075cd710, 0x8019a9, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00e7ebe88 sp=0xc00e7ebe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0075cd710, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00e7ebec8 sp=0xc00e7ebe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00697af80?, {0xc00e498500?, 0x10, 0xc00697af70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00e7ebfa8 sp=0xc00e7ebec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00e7ebfe0 sp=0xc00e7ebfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e7ebfe8 sp=0xc00e7ebfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 476822 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00e8d3640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010a295d0 sp=0xc010a295b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc0075f6510, 0x71) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc010a29618 sp=0xc010a295d0 pc=0x4be30c sync.(*Cond).Wait(0xc00391b000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc010a29650 sp=0xc010a29618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00391b668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc010a29708 sp=0xc010a29650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003452000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc010a29cb8 sp=0xc010a29708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003452000, {0x624a8a0, 0xc0075f6540}, 0x41194d2800000000?, 0xc007ba3dc0?, 0xc00faa2500?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc010a29d70 sp=0xc010a29cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0075f6540?}, {0xc0059fa000?}, 0x3?, 0xc003bf9e90?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc010a29e20 sp=0xc010a29d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc010a29fe0 sp=0xc010a29e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010a29fe8 sp=0xc010a29fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 472460 [sleep]: runtime.gopark(0x4b8d0f6d42d?, 0x400e51?, 0x3e?, 0xfe?, 0x80c75e548f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a627df0 sp=0xc00a627dd0 pc=0x4907d6 time.Sleep(0x15279ea1d) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a627e30 sp=0xc00a627df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00d437a40, 0x400e51, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a627e88 sp=0xc00a627e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00d437a40, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a627ec8 sp=0xc00a627e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00cf23500?, 0x35, 0xc00c823f98?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a627fa8 sp=0xc00a627ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a627fe0 sp=0xc00a627fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a627fe8 sp=0xc00a627fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 120409 [runnable]: runtime.gopark(0xc0210fb950?, 0x4?, 0xe?, 0x1?, 0xc0210fb8a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0210fb700 sp=0xc0210fb6e0 pc=0x4907d6 runtime.selectgo(0xc0210fb950, 0xc0210fb898, 0xc00eb73040?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0210fb840 sp=0xc0210fb700 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000c62700, 0xc0160a9c70, {0x628a900, 0xc0160a9e50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:434 +0x345 fp=0xc0210fbcd8 sp=0xc0210fb840 pc=0x1d83c25 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x624a948, 0xc01743c0f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:601 +0x367 fp=0xc0210fbe18 sp=0xc0210fbcd8 pc=0x1d852e7 runtime/pprof.Do({0x624a948?, 0xc01598f470?}, {{0xc0116e49e0?, 0xc003bddeb8?, 0xc448e6?}}, 0xc0116e48e0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0210fbe88 sp=0xc0210fbe18 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x624a948, 0xc01598f470}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:607 +0x1c8 fp=0xc0210fbf30 sp=0xc0210fbe88 pc=0x1d84f28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0210fbfe0 sp=0xc0210fbf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0210fbfe8 sp=0xc0210fbfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498064 [semacquire]: runtime.gopark(0xae1b760534e?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c06e728 sp=0xc00c06e708 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00c06e790 sp=0xc00c06e728 pc=0x4a17de sync.runtime_SemacquireMutex(0x7?, 0x7?, 0x10?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00c06e7c0 sp=0xc00c06e790 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x1067fe8?, 0xc01575cd80?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00c06e818 sp=0xc00c06e7c0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x24b?, 0x20218?, 0x11?, 0x8c?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00c06e860 sp=0xc00c06e818 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc005b29080, {0x13?, 0x13?}, 0x0, 0xc01839a278, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00c06e960 sp=0xc00c06e860 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc01839a000, 0x11?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00c06ea00 sp=0xc00c06e960 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*compactionIterator).skipForward(0xc003d6e2a0, 0xc0155da000?, {0x0?, 0xc00c06eab0?, 0x19?}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1391 +0x7f fp=0xc00c06ea68 sp=0xc00c06ea00 pc=0x10c6edf github.com/cockroachdb/pebble/sstable.(*compactionIterator).Next(0xc003d6e2a0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1378 +0x5d fp=0xc00c06eaa0 sp=0xc00c06ea68 pc=0x10c6dfd github.com/cockroachdb/pebble.(*levelIter).Next(0xc00c0afb80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:806 +0x167 fp=0xc00c06eac8 sp=0xc00c06eaa0 pc=0x114c207 github.com/cockroachdb/pebble.(*mergingIter).nextEntry(0xc0039d5dc0, 0xc015039e00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:555 +0x6c fp=0xc00c06eb18 sp=0xc00c06eac8 pc=0x1150d8c github.com/cockroachdb/pebble.(*mergingIter).Next(0xc0039d5dc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:1107 +0x4e fp=0xc00c06eb38 sp=0xc00c06eb18 pc=0x115326e github.com/cockroachdb/pebble.(*compactionIter).iterNext(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:473 github.com/cockroachdb/pebble.(*compactionIter).nextInStripe(0xc006e48900) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:496 +0x2e fp=0xc00c06eb88 sp=0xc00c06eb38 pc=0x110df4e github.com/cockroachdb/pebble.(*compactionIter).setNext(0xc006e48900) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:574 +0x27f fp=0xc00c06ebe0 sp=0xc00c06eb88 pc=0x110e43f github.com/cockroachdb/pebble.(*compactionIter).Next(0xc006e48900) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction_iter.go:368 +0x705 fp=0xc00c06ec80 sp=0xc00c06ebe0 pc=0x110d9a5 github.com/cockroachdb/pebble.(*DB).runCompaction(0xc002a2cd80, 0x1328, 0xc00486c000) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2690 +0x1ea5 fp=0xc00c06fc60 sp=0xc00c06ec80 pc=0x1107205 github.com/cockroachdb/pebble.(*DB).compact1(0xc002a2cd80, 0xc00486c000, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2180 +0x1a5 fp=0xc00c06fe80 sp=0xc00c06fc60 pc=0x1104905 github.com/cockroachdb/pebble.(*DB).compact.func1({0x624a948, 0xc013a19320}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2151 +0xad fp=0xc00c06fee8 sp=0xc00c06fe80 pc=0x110468d runtime/pprof.Do({0x624a8d8?, 0xc0001a0000?}, {{0xc000084440?, 0x6294e00?, 0xc0035c51e0?}}, 0xc00a903f88) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00c06ff58 sp=0xc00c06fee8 pc=0xfd3223 github.com/cockroachdb/pebble.(*DB).compact(0xc1db00?, 0xc0100cf380?, 0xc00a903fb8?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2148 +0x6b fp=0xc00c06ffb8 sp=0xc00c06ff58 pc=0x11045ab github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker.func4() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1869 +0x2c fp=0xc00c06ffe0 sp=0xc00c06ffb8 pc=0x1102b0c runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c06ffe8 sp=0xc00c06ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1869 +0x5ea goroutine 479678 [sleep]: runtime.gopark(0x4b8c1ff59da?, 0x781994?, 0x51?, 0x70?, 0x80ab17fc9e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003a6fdf0 sp=0xc003a6fdd0 pc=0x4907d6 time.Sleep(0x1452b0bfa) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003a6fe30 sp=0xc003a6fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc011d1bf20, 0x781994, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003a6fe88 sp=0xc003a6fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc011d1bf20, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003a6fec8 sp=0xc003a6fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc003a6ff80?, {0xc00c8be780?, 0xd, 0xc003a6ff70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003a6ffa8 sp=0xc003a6fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003a6ffe0 sp=0xc003a6ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003a6ffe8 sp=0xc003a6ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498123 [select]: runtime.gopark(0xc00d013e60?, 0x3?, 0x0?, 0x0?, 0xc00d013e32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d013cb8 sp=0xc00d013c98 pc=0x4907d6 runtime.selectgo(0xc00d013e60, 0xc00d013e2c, 0xc010bc6270?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00d013df8 sp=0xc00d013cb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0146fc2d0, {0x624a948, 0xc010bc6270}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc00d013eb0 sp=0xc00d013df8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc010bc6270?}) :1 +0x39 fp=0xc00d013ed8 sp=0xc00d013eb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc010bc6270}, {0x53ddbcd?, 0x1054e20?}, 0xc00a57ffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc00d013f50 sp=0xc00d013ed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc00d013fe0 sp=0xc00d013f50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d013fe8 sp=0xc00d013fe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 45217 [select, 7 minutes]: runtime.gopark(0xc00c001f98?, 0x2?, 0x0?, 0x0?, 0xc00c001f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c001e00 sp=0xc00c001de0 pc=0x4907d6 runtime.selectgo(0xc00c001f98, 0xc00c001f78, 0xc00db4c420?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c001f40 sp=0xc00c001e00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0131572c0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc00c001fc8 sp=0xc00c001f40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc00c001fe0 sp=0xc00c001fc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c001fe8 sp=0xc00c001fe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 49078 [select]: runtime.gopark(0xc00a4d3950?, 0x4?, 0x0?, 0x0?, 0xc00a4d38a0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a4d3700 sp=0xc00a4d36e0 pc=0x4907d6 runtime.selectgo(0xc00a4d3950, 0xc00a4d3898, 0xc0111d1ba0?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a4d3840 sp=0xc00a4d3700 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000c62700, 0xc015e94960, {0x628a900, 0xc015e94a40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:434 +0x345 fp=0xc00a4d3cd8 sp=0xc00a4d3840 pc=0x1d83c25 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x624a948, 0xc006483620}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:601 +0x367 fp=0xc00a4d3e18 sp=0xc00a4d3cd8 pc=0x1d852e7 runtime/pprof.Do({0x624a948?, 0xc006482ba0?}, {{0xc00df7de00?, 0xc0042f3eb8?, 0xc448e6?}}, 0xc00df7ddc0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc00a4d3e88 sp=0xc00a4d3e18 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x624a948, 0xc006482ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:607 +0x1c8 fp=0xc00a4d3f30 sp=0xc00a4d3e88 pc=0x1d84f28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a4d3fe0 sp=0xc00a4d3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a4d3fe8 sp=0xc00a4d3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498488 [select]: runtime.gopark(0xc00ba11208?, 0x2?, 0x70?, 0xa5?, 0xc00ba11204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ba11090 sp=0xc00ba11070 pc=0x4907d6 runtime.selectgo(0xc00ba11208, 0xc00ba11200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ba111d0 sp=0xc00ba11090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc02d8886c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00ba11238 sp=0xc00ba111d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030826580, {0x51b9a20?, 0xc0533f6400}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00ba11340 sp=0xc00ba11238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00ba11370 sp=0xc00ba11340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc02d888360, 0xc00ba11448, 0xc00ba11418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00ba113c8 sp=0xc00ba11370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc02d888360, {0x51b9a20?, 0xc0533f6400?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00ba11478 sp=0xc00ba113c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00564dad0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0172272c0}, {0x51b9a20, 0xc0533f6400}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00ba114e0 sp=0xc00ba11478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00564dad0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0172272c0}, {0x51b9a20, 0xc0533f6400}, 0x170ec5edef015a39?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00ba115d0 sp=0xc00ba114e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00564dad0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0172272c0?}, {0x51b9a20?, 0xc0533f6400?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00ba11670 sp=0xc00ba115d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277390, {0x624a948, 0xc00564dad0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00ba116f0 sp=0xc00ba11670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277390?}}, {0x624a948, 0xc00564dad0}, 0xc0172272c0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00ba117c8 sp=0xc00ba116f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00564dad0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc00ba11818 sp=0xc00ba117c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc010cd93e0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00ba11ab8 sp=0xc00ba11818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc010cd93e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e494dbda, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00ba11bf0 sp=0xc00ba11ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e494dbda, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00ba12338 sp=0xc00ba11bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e494dbda, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00ba12c38 sp=0xc00ba12338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e494dbda, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00ba134a0 sp=0xc00ba12c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e494dbda, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00ba138c0 sp=0xc00ba134a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00ba139f8 sp=0xc00ba138c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00564daa0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00ba13c00 sp=0xc00ba139f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00564daa0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00ba13f30 sp=0xc00ba13c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ba13fe0 sp=0xc00ba13f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ba13fe8 sp=0xc00ba13fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 497800 [select]: runtime.gopark(0xc0108ba588?, 0x3?, 0x42?, 0xa3?, 0xc0108ba4aa?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0108ba260 sp=0xc0108ba240 pc=0x4907d6 runtime.selectgo(0xc0108ba588, 0xc0108ba4a4, 0xc00511d2c0?, 0x0, 0x5?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0108ba3a0 sp=0xc0108ba260 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc00646dc80}, {0x6221538, 0xc01052bfe0}, {0xf0?, {0xc003d96710?, 0x3d965b0?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc0108ba710 sp=0xc0108ba3a0 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:888 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0xc00581a218?, {0x624a948, 0xc00646dc80}, {{{0xc01121c818, 0x5, 0x8}, {0x0, 0x0, 0x0}}, {{0x4f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:881 +0x115 fp=0xc0108ba788 sp=0xc0108ba710 pc=0x1ca8435 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc00b56cf50, {0x624a948, 0xc00646dc80}, {0xc01477ed80, {0x17600a769ef0d4af, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:655 +0x814 fp=0xc0108bad40 sp=0xc0108ba788 pc=0x1cd0fb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc00646dc80?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc0108bae70 sp=0xc0108bad40 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc00b56cf50, {0x624a948, 0xc00646dc80}, {0xc01477ed80, {0x17600a769ef0d4af, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc0108bb558 sp=0xc0108bae70 pc=0x1ccefa6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc00646dc80}, 0xc00ba8adc0, 0xc008929af8?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc0108bb9c8 sp=0xc0108bb558 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00646dc80}, 0xc008929bb0?, {0xc01477ed80, {0x17600a769ef0d4af, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0108bbb08 sp=0xc0108bb9c8 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00646dc80}, 0xc00519f860, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0108bbe28 sp=0xc0108bbb08 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a769ef0d4af, 0x0, 0x0}, 0x0, {0x17600a76a58b9322, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc0108bc1f0 sp=0xc0108bbe28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a769ef0d4af, 0x0, 0x0}, 0x0, {0x17600a76a58b9322, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0108bcb78 sp=0xc0108bc1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76a58b9322, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0108bcdc0 sp=0xc0108bcb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00646dbf0}, {0xc0118e6ac8?}, 0xc00519eb40) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0108bd430 sp=0xc0108bcdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00646db30}, 0xc00519eb40) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0108bd4d8 sp=0xc0108bd430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc00646db30}, {0x52f0080?, 0xc00519eb40}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc0108bd518 sp=0xc0108bd4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00646db30}, {0x52f0080, 0xc00519eb40}, 0xc008a7f760, 0xc00dad1800) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0108bd848 sp=0xc0108bd518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00646db30?}, {0x52f0080?, 0xc00519eb40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc0108bd888 sp=0xc0108bd848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00646db30}, {0x52f0080, 0xc00519eb40}, 0xc00892b920?, 0xc00cf81900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0108bd8b8 sp=0xc0108bd888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00646db30?}, {0x52f0080?, 0xc00519eb40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0108bd8f8 sp=0xc0108bd8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00646db30?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0108bd930 sp=0xc0108bd8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00646db30}, {0x203002?, 0x40?}, 0xc00892b9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0108bd9a8 sp=0xc0108bd930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00646db30?}, {0x52f0080?, 0xc00519eb40?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0108bda20 sp=0xc0108bd9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc00646db30?}, {0x52f0080?, 0xc00519eb40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0108bda60 sp=0xc0108bda20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc00646db30}, {0x52f0080, 0xc00519eb40}, 0xc008a7f760, 0xc00dad1800) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc0108bdab0 sp=0xc0108bda60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc00646db30}, 0xc0183472c0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc0108bdb08 sp=0xc0108bdab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc01477e900, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc0108bde48 sp=0xc0108bdb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc01477e900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc0108bdf68 sp=0xc0108bde48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc0108bdfe0 sp=0xc0108bdf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0108bdfe8 sp=0xc0108bdfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498348 [select]: runtime.gopark(0xc011a31300?, 0x3?, 0x1?, 0x0?, 0xc011a30fca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011a30d50 sp=0xc011a30d30 pc=0x4907d6 runtime.selectgo(0xc011a31300, 0xc011a30fc4, 0x17600a76e0b4de90?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc011a30e90 sp=0xc011a30d50 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc00d93e660}, 0xc01068e000, 0xc005fa34a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc011a31600 sp=0xc011a30e90 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc00d93e660}, 0xc01068e000, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc011a31920 sp=0xc011a31600 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76e0b4de90, 0x0, 0x0}, 0xc0056b2b80, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc011a31ce8 sp=0xc011a31920 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e0b4de90, 0x0, 0x0}, 0xc0056b2b80, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc011a32670 sp=0xc011a31ce8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x624a948?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc011a328b8 sp=0xc011a32670 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc010e106c0}, {0xc006a4a780?}, 0xc004bb7ef0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc011a32f28 sp=0xc011a328b8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc010e10480}, 0xc004bb7ef0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc011a32fd0 sp=0xc011a32f28 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc010e10480?}, {0x52f0080?, 0xc004bb7ef0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc011a33000 sp=0xc011a32fd0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc010e10480}, {0x52f0080, 0xc004bb7ef0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc011a33330 sp=0xc011a33000 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc010e10480?}, {0x52f0080?, 0xc004bb7ef0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc011a33370 sp=0xc011a33330 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc010e10480}, {0x52f0080, 0xc004bb7ef0}, 0xc00b0fb408?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc011a333a0 sp=0xc011a33370 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc010e10480?}, {0x52f0080?, 0xc004bb7ef0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc011a333e0 sp=0xc011a333a0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc010e10480?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc011a33418 sp=0xc011a333e0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc010e10480}, {0xc003ca8000?, 0x3?}, 0xc00b0fb4d0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc011a33490 sp=0xc011a33418 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc010e10480?}, {0x52f0080?, 0xc004bb7ef0?}, 0x51b9a20?, 0xc00418e300?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc011a33508 sp=0xc011a33490 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc010e10480?}, {0x52f0080?, 0xc004bb7ef0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc011a33548 sp=0xc011a33508 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc010e10480?}, {0xc010e10480?, 0x4?}, {0x52f0080?, 0xc004bb7ef0?}, {0x51b9a20?, 0xc00418e380}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc011a33588 sp=0xc011a33548 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc010e10480}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc004bb7ef0}, {0x51b9a20, 0xc00418e380}, 0xc00b0fb700?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc011a33678 sp=0xc011a33588 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc010e10480}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc004bb7ef0}, {0x51b9a20, 0xc00418e380}, 0xc00b0fb7e0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc011a33710 sp=0xc011a33678 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc010e10420}, 0xc004bb7ef0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc011a33790 sp=0xc011a33710 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xb0fb830?, {0x624a948?, 0xc010e10420?}, 0x624a948?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc011a33818 sp=0xc011a33790 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0125d0840, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc011a33ab8 sp=0xc011a33818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0125d0840, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc011a33bf0 sp=0xc011a33ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc011a34338 sp=0xc011a33bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc011a34c38 sp=0xc011a34338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc011a354a0 sp=0xc011a34c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b3ce95, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc011a358c0 sp=0xc011a354a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc011a359f8 sp=0xc011a358c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc010adce40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc011a35c00 sp=0xc011a359f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc010adce40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc011a35f30 sp=0xc011a35c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011a35fe0 sp=0xc011a35f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011a35fe8 sp=0xc011a35fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 120617 [select, 6 minutes]: runtime.gopark(0xc003a79fb0?, 0x2?, 0x80?, 0xd8?, 0xc003a79f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c24de28 sp=0xc00c24de08 pc=0x4907d6 runtime.selectgo(0xc00c24dfb0, 0xc003a79f98, 0xc002a40360?, 0x0, 0xc0068047d0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c24df68 sp=0xc00c24de28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc00c24dfe0 sp=0xc00c24df68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c24dfe8 sp=0xc00c24dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 433483 [sleep]: runtime.gopark(0x4b8f1c224ae?, 0x205ad5?, 0x2e?, 0xa0?, 0x80e8b6f68d?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003a69df0 sp=0xc003a69dd0 pc=0x4907d6 time.Sleep(0x153073a9d) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003a69e30 sp=0xc003a69df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0289f9c50, 0x205ad5, {0x12?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003a69e88 sp=0xc003a69e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0289f9c50, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003a69ec8 sp=0xc003a69e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc008675f80?, {0xc00ce8f500?, 0x22, 0xc008675f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003a69fa8 sp=0xc003a69ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003a69fe0 sp=0xc003a69fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003a69fe8 sp=0xc003a69fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 488453 [runnable]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0xf4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e29ef58 sp=0xc00e29ef38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539900?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00e29ef90 sp=0xc00e29ef58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539818, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00e29efb0 sp=0xc00e29ef90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0084fb280?, 0xc00176d000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00e29efd8 sp=0xc00e29efb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0084fb280, {0xc00176d000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00e29f058 sp=0xc00e29efd8 pc=0x535efa net.(*netFD).Read(0xc0084fb280, {0xc00176d000?, 0x7b5a68?, 0xc0084fb280?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00e29f0a0 sp=0xc00e29f058 pc=0x7a20c9 net.(*conn).Read(0xc003731f40, {0xc00176d000?, 0x164290f?, 0xc003731f40?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00e29f0e8 sp=0xc00e29f0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00dc48d20, {0xc00176d000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00e29f130 sp=0xc00e29f0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0138f7188, {0xc00176d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00e29f1d0 sp=0xc00e29f130 pc=0x370f896 bufio.(*Reader).fill(0xc003918090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00e29f208 sp=0xc00e29f1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003918090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00e29f228 sp=0xc00e29f208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003918668?, {0x62295c8, 0xc003918090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00e29f260 sp=0xc00e29f228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003918000, 0xc00ba6c660, 0xc00e29f442, 0xc00e29f444, {0x623b718, 0xc015084a20}, 0xc0059fa000, 0xc011df40d4, 0xc00e29f440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00e29f398 sp=0xc00e29f260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003918000, {0x624a948, 0xc0047baf60}, 0xc00ba6c650, 0xc0059fa000, 0xc0047bae70, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00e29f768 sp=0xc00e29f398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0047baf60}, {0x628a538, 0xc00dc48d20}, {{{0xc003188009, 0x4}}, 0x1, 0xc0047baea0, 0xc0047baed0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00e29f8a8 sp=0xc00e29f768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0047baf60}, {0x628a538?, 0xc00dc48d20}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00e29fe20 sp=0xc00e29f8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0047bad20?}, {0x628a538, 0xc00dc48d20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00e29feb0 sp=0xc00e29fe20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0047bad20}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00e29ff30 sp=0xc00e29feb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00e29ffe0 sp=0xc00e29ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e29ffe8 sp=0xc00e29ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498516 [select]: runtime.gopark(0xc012175208?, 0x2?, 0x90?, 0xb5?, 0xc012175204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012175090 sp=0xc012175070 pc=0x4907d6 runtime.selectgo(0xc012175208, 0xc012175200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0121751d0 sp=0xc012175090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0052ee360) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc012175238 sp=0xc0121751d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030827810, {0x51b9a20?, 0xc0533f6d80}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc012175340 sp=0xc012175238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc012175370 sp=0xc012175340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0052ee120, 0xc012175448, 0xc012175418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0121753c8 sp=0xc012175370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0052ee120, {0x51b9a20?, 0xc0533f6d80?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc012175478 sp=0xc0121753c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03234bc50?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003bee2d0}, {0x51b9a20, 0xc0533f6d80}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0121754e0 sp=0xc012175478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03234bc50}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003bee2d0}, {0x51b9a20, 0xc0533f6d80}, 0x27a5548b632131ef?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0121755d0 sp=0xc0121754e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc03234bc50?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003bee2d0?}, {0x51b9a20?, 0xc0533f6d80?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc012175670 sp=0xc0121755d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277600, {0x624a948, 0xc03234bc50}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0121756f0 sp=0xc012175670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277600?}}, {0x624a948, 0xc03234bc50}, 0xc003bee2d0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0121757c8 sp=0xc0121756f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc03234bc50?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc012175818 sp=0xc0121757c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00cde7ec0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc012175ab8 sp=0xc012175818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00cde7ec0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e516cb2a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc012175bf0 sp=0xc012175ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e516cb2a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc012176338 sp=0xc012175bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e516cb2a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc012176c38 sp=0xc012176338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e516cb2a, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0121774a0 sp=0xc012176c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e516cb2a, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0121778c0 sp=0xc0121774a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0121779f8 sp=0xc0121778c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc03234bb90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc012177c00 sp=0xc0121779f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc03234bb90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc012177f30 sp=0xc012177c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012177fe0 sp=0xc012177f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012177fe8 sp=0xc012177fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498494 [select]: runtime.gopark(0xc011547208?, 0x2?, 0x90?, 0xb2?, 0xc011547204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011547090 sp=0xc011547070 pc=0x4907d6 runtime.selectgo(0xc011547208, 0xc011547200, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0115471d0 sp=0xc011547090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc058a597a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc011547238 sp=0xc0115471d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030827600, {0x51b9a20?, 0xc0533f6c80}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc011547340 sp=0xc011547238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x11b?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc011547370 sp=0xc011547340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc058a590e0, 0xc011547448, 0xc011547418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0115473c8 sp=0xc011547370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc058a590e0, {0x51b9a20?, 0xc0533f6c80?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc011547478 sp=0xc0115473c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03234a5a0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003bee000}, {0x51b9a20, 0xc0533f6c80}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0115474e0 sp=0xc011547478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03234a5a0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003bee000}, {0x51b9a20, 0xc0533f6c80}, 0x37d20ecb228612ee?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0115475d0 sp=0xc0115474e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc03234a5a0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003bee000?}, {0x51b9a20?, 0xc0533f6c80?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc011547670 sp=0xc0115475d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0432775b0, {0x624a948, 0xc03234a5a0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0115476f0 sp=0xc011547670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0432775b0?}}, {0x624a948, 0xc03234a5a0}, 0xc003bee000, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0115477c8 sp=0xc0115476f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc03234a5a0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc011547818 sp=0xc0115477c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00d1deba0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc011547ab8 sp=0xc011547818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00d1deba0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5141cb0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc011547bf0 sp=0xc011547ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5141cb0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc011548338 sp=0xc011547bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5141cb0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc011548c38 sp=0xc011548338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5141cb0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0115494a0 sp=0xc011548c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5141cb0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0115498c0 sp=0xc0115494a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0115499f8 sp=0xc0115498c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00aeb2270}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc011549c00 sp=0xc0115499f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00aeb2270}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc011549f30 sp=0xc011549c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011549fe0 sp=0xc011549f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011549fe8 sp=0xc011549fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 409521 [sleep]: runtime.gopark(0x4b8d236cd1a?, 0x400ea1?, 0x8b?, 0x41?, 0x80c93b97de?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a2c1df0 sp=0xc00a2c1dd0 pc=0x4907d6 time.Sleep(0x1531733b5) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a2c1e30 sp=0xc00a2c1df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00fb7a270, 0x400ea1, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a2c1e88 sp=0xc00a2c1e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00fb7a270, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a2c1ec8 sp=0xc00a2c1e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc004aa0f80?, {0xc003e48000?, 0x23, 0xc004aa0f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a2c1fa8 sp=0xc00a2c1ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a2c1fe0 sp=0xc00a2c1fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a2c1fe8 sp=0xc00a2c1fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 423631 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x2ce?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b85cf58 sp=0xc00b85cf38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f300678?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00b85cf90 sp=0xc00b85cf58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300590, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00b85cfb0 sp=0xc00b85cf90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0151e9d80?, 0xc00e493000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00b85cfd8 sp=0xc00b85cfb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0151e9d80, {0xc00e493000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00b85d058 sp=0xc00b85cfd8 pc=0x535efa net.(*netFD).Read(0xc0151e9d80, {0xc00e493000?, 0x7b5a68?, 0xc0151e9d80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00b85d0a0 sp=0xc00b85d058 pc=0x7a20c9 net.(*conn).Read(0xc0112a2370, {0xc00e493000?, 0x164290f?, 0xc0112a2370?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00b85d0e8 sp=0xc00b85d0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00f74c000, {0xc00e493000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00b85d130 sp=0xc00b85d0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0143df950, {0xc00e493000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00b85d1d0 sp=0xc00b85d130 pc=0x370f896 bufio.(*Reader).fill(0xc0118e6490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00b85d208 sp=0xc00b85d1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0118e6490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00b85d228 sp=0xc00b85d208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0118e6a68?, {0x62295c8, 0xc0118e6490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00b85d260 sp=0xc00b85d228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0118e6400, 0xc0143e9380, 0xc00b85d442, 0xc00b85d444, {0x623b718, 0xc0060a8e10}, 0xc0059fa000, 0xc0118f2144, 0xc00b85d440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00b85d398 sp=0xc00b85d260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0118e6400, {0x624a948, 0xc008ddc0c0}, 0xc0143e9370, 0xc0059fa000, 0xc008ddc000, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00b85d768 sp=0xc00b85d398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc008ddc0c0}, {0x628a538, 0xc00f74c000}, {{{0xc00512a038, 0x4}}, 0x1, 0xc008ddc030, 0xc008ddc060, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00b85d8a8 sp=0xc00b85d768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc008ddc0c0}, {0x628a538?, 0xc00f74c000}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00b85de20 sp=0xc00b85d8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00ce6fef0?}, {0x628a538, 0xc00f74c000}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00b85deb0 sp=0xc00b85de20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00ce6fef0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00b85df30 sp=0xc00b85deb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b85dfe0 sp=0xc00b85df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b85dfe8 sp=0xc00b85dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 344462 [select]: runtime.gopark(0xc0210e4758?, 0x2?, 0x28?, 0xd7?, 0xc0210e4754?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0210e45e0 sp=0xc0210e45c0 pc=0x4907d6 runtime.selectgo(0xc0210e4758, 0xc0210e4750, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0210e4720 sp=0xc0210e45e0 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc047d0cc60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0210e4788 sp=0xc0210e4720 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00844c9a0, {0x51b9a20?, 0xc013663780}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0210e4890 sp=0xc0210e4788 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xee?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0210e48c0 sp=0xc0210e4890 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc047d0ca20, 0xc0210e4998, 0xc0210e4968) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0210e4918 sp=0xc0210e48c0 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc047d0ca20, {0x51b9a20?, 0xc013663780?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0210e49c8 sp=0xc0210e4918 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc011881920?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00dcec4b0}, {0x51b9a20, 0xc013663780}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0210e4a30 sp=0xc0210e49c8 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011881920}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00dcec4b0}, {0x51b9a20, 0xc013663780}, 0x20fed91093093d0e?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0210e4b20 sp=0xc0210e4a30 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x34fe92a92161e64c?, {0x624a948?, 0xc011881920?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00dcec4b0?}, {0x51b9a20?, 0xc013663780?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc0210e4bc0 sp=0xc0210e4b20 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043276f90, {0x624a948, 0xc011881920}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0210e4c40 sp=0xc0210e4bc0 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043276f90?}}, {0x624a948, 0xc011881920}, 0xc00dcec4b0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0210e4d18 sp=0xc0210e4c40 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x210e4d80?, {0x624a948?, 0xc011881920?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc0210e4e78?}) :1 +0x65 fp=0xc0210e4d68 sp=0xc0210e4d18 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc047139c80, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0210e5008 sp=0xc0210e4d68 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc047139c80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3699e01, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0210e5140 sp=0xc0210e5008 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3699e01, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0210e5888 sp=0xc0210e5140 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3699e01, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0210e6188 sp=0xc0210e5888 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3699e01, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0210e69f0 sp=0xc0210e6188 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3699e01, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0210e6e10 sp=0xc0210e69f0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0146fc638, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc0210e7068 sp=0xc0210e6e10 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0146fc600, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc0210e7190 sp=0xc0210e7068 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0146fc5d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc0210e7420 sp=0xc0210e7190 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0146fc4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc0210e79c0 sp=0xc0210e7420 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0146fc4d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc0210e7cb0 sp=0xc0210e79c0 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0146fc3a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc0210e7fb0 sp=0xc0210e7cb0 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0146fc380?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc0210e8158 sp=0xc0210e7fb0 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0146fc2d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc0210e8370 sp=0xc0210e8158 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0146fc100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc0210e85c8 sp=0xc0210e8370 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc0210e86f0 sp=0xc0210e85c8 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc005ba40b0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc0210e89b8 sp=0xc0210e86f0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc0210e8ad0 sp=0xc0210e89b8 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00e5f8d80, {0x624a948, 0xc011881710}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc0210e8cf0 sp=0xc0210e8ad0 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00e5f8d80, {0x624a948, 0xc011881710}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc0210e8fe8 sp=0xc0210e8cf0 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00d1df980, {0x624a948, 0xc011881710}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc0210e90c8 sp=0xc0210e8fe8 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc008e5f180, {0x624a948, 0xc011881710}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc0210e93a0 sp=0xc0210e90c8 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc00fa0fc20) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc0210e93f8 sp=0xc0210e93a0 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc008d8d800) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc0210e9410 sp=0xc0210e93f8 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc00d1df9e0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f fp=0xc0210e94d0 sp=0xc0210e9410 pc=0x293bf5f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc00e7f13f0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc0210e94e8 sp=0xc0210e94d0 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc0210e9500 sp=0xc0210e94e8 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc006f57c00?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc0210e9540 sp=0xc0210e9500 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc00e7f13f0, {0x624a8a0?, 0xc008d8d7c0?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc0210e96d0 sp=0xc0210e9540 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0082b3cc0, {0x624a8a0, 0xc008d8d7c0}, 0xc005ba40b0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc0210e9770 sp=0xc0210e96d0 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc011881260}, 0xc005423e00, 0xc005ba40b0, 0xc013663680, 0xc006912380, 0xc0042ce1b0, 0xc0153331a0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc0210ea020 sp=0xc0210e9770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc011881260}, 0xe8?, 0xc005423e00, 0x1755c670a9a7ac01?, {{0x624ce40?, 0xc011a13f40?}, 0x0?}, 0xc006912380, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0210ea0e0 sp=0xc0210ea020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc011881260}, 0xc0042ce1b0, 0xc0042cde78?, 0xc0042cde78, 0xc006912380, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc0210ea1f8 sp=0xc0210ea0e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0042cd900, {0x624a948?, 0xc011881260}, 0xc0042cde78, 0xc011881260?, {0x7f275f329680?, 0xc003864ec8}, 0x68?, 0xc008a8b948) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc0210ea300 sp=0xc0210ea1f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0042cd900, {0x624a8a0, 0xc008d8d400}, 0xc0042cde78, {0x7f275f329680, 0xc003864ec8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc0210ea720 sp=0xc0210ea300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0042cd900, {0x624a948?, 0xc04267a9c0?}, {{0x6273700, 0xc00b1d1950}, {0xc002f66be2, 0x66}, 0x3, 0x1}, 0xc0123ba480, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0210eb0b0 sp=0xc0210ea720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc04267a9c0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc0210eb168 sp=0xc0210eb0b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc04267a9c0?}, {0x6273700?, 0xc00b1d1950?}, 0x624a948?, 0xc04267a9c0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0210eb2b8 sp=0xc0210eb168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0042cd900, {0x624a948, 0xc04267a9c0}, {{0x6273700, 0xc00b1d1950}, {0xc002f66be2, 0x66}, 0x3, 0x1}, 0xc0123ba480, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc0210eb430 sp=0xc0210eb2b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0042cd900, {0x624a948, 0xc04267a9c0}, {0xc0123ba480, {0xc0082e7710, 0x3, 0x3}, {0xc0038ea076, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc0210eb550 sp=0xc0210eb430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc005d19bfb, 0x0}, 0x0, {0xc1116e819641975e, 0x7fec30c5ac, 0x0}, 0x1}, 0xc0042cd900, 0xc0210eb8c0, 0xc0210eb8b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc0210eb708 sp=0xc0210eb550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0042cd900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc0210ebcb8 sp=0xc0210eb708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0042cd900, {0x624a8a0, 0xc00f70a4c0}, 0x41190b4000000000?, 0xc003dcfdc0?, 0xc010cbe8c0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0210ebd70 sp=0xc0210ebcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00f70a4c0?}, {0xc0059fa000?}, 0x3?, 0xc0030bc0a8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0210ebe20 sp=0xc0210ebd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0210ebfe0 sp=0xc0210ebe20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0210ebfe8 sp=0xc0210ebfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 474098 [sleep]: runtime.gopark(0x4b889f0904f?, 0x1011bee?, 0xd1?, 0x5c?, 0x80609d1f2e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007ba3df0 sp=0xc007ba3dd0 pc=0x4907d6 time.Sleep(0x132bef7c9) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc007ba3e30 sp=0xc007ba3df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00859c210, 0x1011bee, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc007ba3e88 sp=0xc007ba3e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00859c210, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc007ba3ec8 sp=0xc007ba3e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00e6ee500?, 0xa, 0xc001838fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc007ba3fa8 sp=0xc007ba3ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc007ba3fe0 sp=0xc007ba3fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007ba3fe8 sp=0xc007ba3fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 49041 [select]: runtime.gopark(0xc004205a08?, 0x2?, 0xc0?, 0x8?, 0xc0042059ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004205870 sp=0xc004205850 pc=0x4907d6 runtime.selectgo(0xc004205a08, 0xc0042059e8, 0x16e9c57?, 0x0, 0x2?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0042059b0 sp=0xc004205870 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002cfedc0, {0xc005644628, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc004205a38 sp=0xc0042059b0 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002cfedc0, {0xc005644628?, 0xc003293878?, 0xc004205a80?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc004205a78 sp=0xc004205a38 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc006bcdb90, {0xc005644628?, 0xc004205af8?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc004205ac0 sp=0xc004205a78 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc006bcdb90}, {0xc005644628, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc004205b08 sp=0xc004205ac0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00417d320, {0xc005644628, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc004205b48 sp=0xc004205b08 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc005644618, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc004205bc8 sp=0xc004205b48 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x0?, 0xc00417d320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc004205c60 sp=0xc004205bc8 pc=0xc19546 google.golang.org/grpc.recv(0x1?, {0x7f275f301038, 0xc0000af170}, 0x468405?, {0x0?, 0x0?}, {0x512da60, 0xc015b2fde8}, 0x13f?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc004205cd8 sp=0xc004205c60 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00a7fad00, {0x512da60?, 0xc015b2fde8}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc004205e08 sp=0xc004205cd8 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x4618e7?, {0x512da60?, 0xc015b2fde8?}) :1 +0x34 fp=0xc004205e30 sp=0xc004205e08 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00d059ee0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:263 +0x4c fp=0xc004205e60 sp=0xc004205e30 pc=0x1d4b66c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:91 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x628a958?, 0xc00d059ee0}, 0xc000c62700, {0x624a948, 0xc00380b080}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:289 +0xa6 fp=0xc004205ee0 sp=0xc004205e60 pc=0x1d82d66 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x624a948?, 0xc00380b080?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:308 +0x45 fp=0xc004205f30 sp=0xc004205ee0 pc=0x1d82c65 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc004205fe0 sp=0xc004205f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004205fe8 sp=0xc004205fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49170 [select, 7 minutes]: runtime.gopark(0xc008b09578?, 0x2?, 0x20?, 0xfb?, 0xc008b0953c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00853b3a8 sp=0xc00853b388 pc=0x4907d6 runtime.selectgo(0xc00853b578, 0xc008b09538, 0xc011e7bd80?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00853b4e8 sp=0xc00853b3a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000c62700, {0x628a958, 0xc00df462e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:313 +0x208 fp=0xc00853b5b0 sp=0xc00853b4e8 pc=0x1d82b08 github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x525f4e0?, 0xc000c62700}, {0x627d570?, 0xc01237a440}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:244 +0x9f fp=0xc00853b5e8 sp=0xc00853b5b0 pc=0x1d4b57f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc00a7fb450?}, 0xc005645878, 0x5683c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc00853b920 sp=0xc00853b5e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc00a7fb450?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc00853b960 sp=0xc00853b920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc00a7fb450}, 0x1054f90?, 0xc011e7bd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc00853b990 sp=0xc00853b960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc00a7fb450?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc00853b9d0 sp=0xc00853b990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc00853ba00 sp=0xc00853b9d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc004bd7350}, {0xc008b09b08?, 0x46159f?}, 0xc008b09ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00853ba78 sp=0xc00853ba00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc00a7fb450?}, 0xc005645878, 0xc011e7bd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc00853bb00 sp=0xc00853ba78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc00a7fb450?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc00853bb40 sp=0xc00853bb00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc00a7fb450}, 0xc005645878, 0x5683c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc00853bb90 sp=0xc00853bb40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc004960d80, 0xc000ace270, 0x8ff1fe0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc00853be48 sp=0xc00853bb90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc004960d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc00853bf68 sp=0xc00853be48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00853bfe0 sp=0xc00853bf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00853bfe8 sp=0xc00853bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 344545 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00b6cb640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01141b5d0 sp=0xc01141b5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00f999b50, 0x235) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc01141b618 sp=0xc01141b5d0 pc=0x4be30c sync.(*Cond).Wait(0xc00dd12c00?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc01141b650 sp=0xc01141b618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00dd13268) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc01141b708 sp=0xc01141b650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003bba000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc01141bcb8 sp=0xc01141b708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003bba000, {0x624a8a0, 0xc00f999b80}, 0x411ac6e000000000?, 0xc006ae9dc0?, 0xc00771a480?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc01141bd70 sp=0xc01141bcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00f999b80?}, {0xc0059fa000?}, 0x3?, 0xc001c10cc0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc01141be20 sp=0xc01141bd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc01141bfe0 sp=0xc01141be20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01141bfe8 sp=0xc01141bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 45301 [select]: runtime.gopark(0xc001b28f48?, 0x2?, 0x6?, 0x0?, 0xc001b28ef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b28d50 sp=0xc001b28d30 pc=0x4907d6 runtime.selectgo(0xc001b28f48, 0xc001b28ef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001b28e90 sp=0xc001b28d50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00db2d4a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc001b28fc8 sp=0xc001b28e90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc001b28fe0 sp=0xc001b28fc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b28fe8 sp=0xc001b28fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 497924 [select]: runtime.gopark(0xc0086b5e60?, 0x3?, 0x0?, 0x0?, 0xc0086b5e32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0086b5cb8 sp=0xc0086b5c98 pc=0x4907d6 runtime.selectgo(0xc0086b5e60, 0xc0086b5e2c, 0xc019df80f0?, 0x0, 0xc003dcfe40?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0086b5df8 sp=0xc0086b5cb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc016880cd0, {0x624a948, 0xc019df80f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc0086b5eb0 sp=0xc0086b5df8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc019df80f0?}) :1 +0x39 fp=0xc0086b5ed8 sp=0xc0086b5eb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc019df80f0}, {0x53ddbcd?, 0x1054e20?}, 0xc003dcffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc0086b5f50 sp=0xc0086b5ed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc0086b5fe0 sp=0xc0086b5f50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0086b5fe8 sp=0xc0086b5fe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 422954 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1fa?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dfa2f58 sp=0xc00dfa2f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7810?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00dfa2f90 sp=0xc00dfa2f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7728, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00dfa2fb0 sp=0xc00dfa2f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc006ad0d80?, 0xc002f30000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00dfa2fd8 sp=0xc00dfa2fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc006ad0d80, {0xc002f30000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00dfa3058 sp=0xc00dfa2fd8 pc=0x535efa net.(*netFD).Read(0xc006ad0d80, {0xc002f30000?, 0x7b5a68?, 0xc006ad0d80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00dfa30a0 sp=0xc00dfa3058 pc=0x7a20c9 net.(*conn).Read(0xc0123bec10, {0xc002f30000?, 0x164290f?, 0xc0123bec10?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00dfa30e8 sp=0xc00dfa30a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00f124f60, {0xc002f30000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00dfa3130 sp=0xc00dfa30e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01383c9d8, {0xc002f30000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00dfa31d0 sp=0xc00dfa3130 pc=0x370f896 bufio.(*Reader).fill(0xc00f35bc90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00dfa3208 sp=0xc00dfa31d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00f35bc90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00dfa3228 sp=0xc00dfa3208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00f35c268?, {0x62295c8, 0xc00f35bc90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00dfa3260 sp=0xc00dfa3228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00f35bc00, 0xc012353f70, 0xc00dfa3442, 0xc00dfa3444, {0x623b718, 0xc0381d8630}, 0xc0059fa000, 0xc00e4b13e4, 0xc00dfa3440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00dfa3398 sp=0xc00dfa3260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00f35bc00, {0x624a948, 0xc0171f1110}, 0xc012353f60, 0xc0059fa000, 0xc0171f1050, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00dfa3768 sp=0xc00dfa3398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0171f1110}, {0x628a538, 0xc00f124f60}, {{{0xc00b9c6038, 0x4}}, 0x1, 0xc0171f1080, 0xc0171f10b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00dfa38a8 sp=0xc00dfa3768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0171f1110}, {0x628a538?, 0xc00f124f60}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00dfa3e20 sp=0xc00dfa38a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0171f0d50?}, {0x628a538, 0xc00f124f60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00dfa3eb0 sp=0xc00dfa3e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0171f0d50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00dfa3f30 sp=0xc00dfa3eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00dfa3fe0 sp=0xc00dfa3f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dfa3fe8 sp=0xc00dfa3fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 481894 [sleep]: runtime.gopark(0x4b8bd9d64e2?, 0x1eee8df?, 0x28?, 0x3?, 0x809eaf8f76?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00462ddf0 sp=0xc00462ddd0 pc=0x4907d6 time.Sleep(0x13d249354) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00462de30 sp=0xc00462ddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc007853a70, 0x1eee8df, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00462de88 sp=0xc00462de30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc007853a70, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00462dec8 sp=0xc00462de88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00f48c640?, 0x8, 0xc00157bfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00462dfa8 sp=0xc00462dec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00462dfe0 sp=0xc00462dfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00462dfe8 sp=0xc00462dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498436 [runnable]: runtime.gopark(0xae1b6288cab?, 0x0?, 0x2b?, 0x8c?, 0xc004c13210?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ae29498 sp=0xc00ae29478 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x10?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00ae29500 sp=0xc00ae29498 pc=0x4a17de sync.runtime_SemacquireMutex(0x23?, 0x60?, 0x184d4b2?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00ae29530 sp=0xc00ae29500 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00ae29588 sp=0xc00ae29530 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x1801f9e?, 0x7f270c2d5aa0?, 0x3cc5?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00ae295d0 sp=0xc00ae29588 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00858db80, {0x8?, 0x11?}, 0x0, 0x0, 0xc004c13210) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00ae296d0 sp=0xc00ae295d0 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2727 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc00c710900, {0xc003589620, 0x11, 0x30}, {0xc00522cf30, 0x11, 0x11}, 0x60?, 0x61?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:847 +0xf8 fp=0xc00ae29738 sp=0xc00ae296d0 pc=0x10c5078 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc004c14340?, {0xc003589620?, 0x1?, 0x11?}, {0xc00522cf30?, 0x10?, 0x1153b23?}, 0x18?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00ae29788 sp=0xc00ae29738 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc004c14340, {0xc003589620, 0x11, 0x30}, {0xc00522cf30, 0x11, 0x11}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00ae297d8 sp=0xc00ae29788 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc004c13420, {0xc00522cf30?, 0x0?, 0x0?}, 0x0?, 0xed?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00ae29908 sp=0xc00ae297d8 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc004c13420, {0xc003589620?, 0x11?, 0xc000ab6138?}, {0xc00522cf30?, 0x18?, 0x10?}, 0x10?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00ae29948 sp=0xc00ae29908 pc=0x115285d github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc004c12f00, {0xc00522cf30, 0x11, 0x11}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00ae299e0 sp=0xc00ae29948 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGE(0xc00c34c938, {{0xc010cb9040, 0x10, 0x10}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:347 +0xd3 fp=0xc00ae29a38 sp=0xc00ae299e0 pc=0x1862f13 github.com/cockroachdb/cockroach/pkg/storage.GetIntent({0x628f760, 0xc00c34c680}, {0xc0054c7760, 0xa, 0x10}) github.com/cockroachdb/cockroach/pkg/storage/engine.go:1120 +0x1f6 fp=0xc00ae29ec8 sp=0xc00ae29a38 pc=0x1806d56 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryIntent({_, _}, {_, _}, {{0x62cc5e0, 0xc0075fcd50}, {{0x17600a76b7272fa9, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_intent.go:79 +0x2a6 fp=0xc00ae2a108 sp=0xc00ae29ec8 pc=0x1cfabc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76b7272fa9, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:522 +0x427 fp=0xc00ae2a998 sp=0xc00ae2a108 pc=0x1dc63a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0054225a0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00ae2c808 sp=0xc00ae2a998 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00ae2d278, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc00ae2ce78 sp=0xc00ae2c808 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc00737dc00, {0x624a948, 0xc018c52ed0}, 0xc0054225a0, 0xc002dda140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc00ae2db08 sp=0xc00ae2ce78 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00737dc00, {0x624a948, 0xc018c52ed0}, 0xc0054225a0, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00ae2de28 sp=0xc00ae2db08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00737dc00, {_, _}, {{{0x17600a76b7272fa9, 0x0, 0x0}, 0x0, {0x17600a76e16e3d12, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00ae2e1f0 sp=0xc00ae2de28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76b7272fa9, 0x0, 0x0}, 0x0, {0x17600a76e16e3d12, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00ae2eb78 sp=0xc00ae2e1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e16e3d12, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00ae2edc0 sp=0xc00ae2eb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc018c52e70}, {0xc00becf508?}, 0xc0054223c0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00ae2f430 sp=0xc00ae2edc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc018c52db0}, 0xc0054223c0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00ae2f4d8 sp=0xc00ae2f430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc018c52db0}, {0x52f0080?, 0xc0054223c0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00ae2f518 sp=0xc00ae2f4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc018c52db0}, {0x52f0080, 0xc0054223c0}, 0xc010c70460, 0xc013dccf48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00ae2f848 sp=0xc00ae2f518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c52db0?}, {0x52f0080?, 0xc0054223c0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00ae2f888 sp=0xc00ae2f848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc018c52db0}, {0x52f0080, 0xc0054223c0}, 0xc00ae2f920?, 0xc008d8cbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00ae2f8b8 sp=0xc00ae2f888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c52db0?}, {0x52f0080?, 0xc0054223c0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00ae2f8f8 sp=0xc00ae2f8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc018c52db0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00ae2f930 sp=0xc00ae2f8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc018c52db0}, {0x203004?, 0x40?}, 0xc00ae2f9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00ae2f9a8 sp=0xc00ae2f930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc018c52db0?}, {0x52f0080?, 0xc0054223c0?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00ae2fa20 sp=0xc00ae2f9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c52db0?}, {0x52f0080?, 0xc0054223c0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00ae2fa60 sp=0xc00ae2fa20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc018c52db0}, {0x52f0080, 0xc0054223c0}, 0xc010c70460, 0xc013dccf48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00ae2fab0 sp=0xc00ae2fa60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc018c52db0}, 0xc00d1de840, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00ae2fb08 sp=0xc00ae2fab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc008148fc0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00ae2fe48 sp=0xc00ae2fb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc008148fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00ae2ff68 sp=0xc00ae2fe48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00ae2ffe0 sp=0xc00ae2ff68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ae2ffe8 sp=0xc00ae2ffe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 87449 [semacquire, 6 minutes]: runtime.gopark(0xc00a88fa68?, 0x46159f?, 0x0?, 0x5b?, 0x1c011b4a1b0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003fa99f0 sp=0xc003fa99d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc006c4ea10, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc003fa9a58 sp=0xc003fa99f0 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc003fa9a88 sp=0xc003fa9a58 pc=0x4be005 sync.(*WaitGroup).Wait(0xc00a88fae8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc003fa9ab0 sp=0xc003fa9a88 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc006c4ea00) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc003fa9ac8 sp=0xc003fa9ab0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc006c4ea00?, {0x624a8a0?, 0xc006c4e9c0?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc003fa9af8 sp=0xc003fa9ac8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x624a948?, 0xc000a9c780?}, {0xc00a88fc68, 0x2, 0xc000a9c780?}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 fp=0xc003fa9b48 sp=0xc003fa9af8 pc=0x1187e11 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc006663b80, {0x624a948, 0xc000a9c780}, 0xc003f562c0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:311 +0x6cd fp=0xc003fa9e68 sp=0xc003fa9b48 pc=0x1a00e8d github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x624a948, 0xc010d81f50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 fp=0xc003fa9f30 sp=0xc003fa9e68 pc=0x1a00479 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003fa9fe0 sp=0xc003fa9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003fa9fe8 sp=0xc003fa9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49040 [select, 7 minutes]: runtime.gopark(0xc008503578?, 0x2?, 0x20?, 0xfb?, 0xc00850353c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00853d3a8 sp=0xc00853d388 pc=0x4907d6 runtime.selectgo(0xc00853d578, 0xc008503538, 0xc011e7b480?, 0x0, 0x27?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00853d4e8 sp=0xc00853d3a8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000c62700, {0x628a958, 0xc00d059ee0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:313 +0x208 fp=0xc00853d5b0 sp=0xc00853d4e8 pc=0x1d82b08 github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x525f4e0?, 0xc000c62700}, {0x627d570?, 0xc003287ca0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:244 +0x9f fp=0xc00853d5e8 sp=0xc00853d5b0 pc=0x1d4b57f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc00a7fad00?}, 0xc005644630, 0x5683c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc00853d920 sp=0xc00853d5e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc00a7fad00?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc00853d960 sp=0xc00853d920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc00a7fad00}, 0x1054f90?, 0xc011e7b300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc00853d990 sp=0xc00853d960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc00a7fad00?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc00853d9d0 sp=0xc00853d990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc00853da00 sp=0xc00853d9d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc006bcdec0}, {0xc008503b08?, 0x46159f?}, 0xc008503ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00853da78 sp=0xc00853da00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x525f4e0, 0xc000c62700}, {0x627dc78?, 0xc00a7fad00?}, 0xc005644630, 0xc011e7b300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc00853db00 sp=0xc00853da78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x525f4e0?, 0xc000c62700?}, {0x627dc78?, 0xc00a7fad00?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc00853db40 sp=0xc00853db00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x525f4e0, 0xc000c62700}, {0x627dc78, 0xc00a7fad00}, 0xc005644630, 0x5683c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc00853db90 sp=0xc00853db40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc0074271e0}, 0xc00417d320, 0xc000ace270, 0x8ff1fe0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc00853de48 sp=0xc00853db90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0074271e0}, 0xc00417d320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc00853df68 sp=0xc00853de48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00853dfe0 sp=0xc00853df68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00853dfe8 sp=0xc00853dfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 115795 [select]: runtime.gopark(0xc00ba80d68?, 0x2?, 0x0?, 0xa6?, 0xc00ba80d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ba80bb8 sp=0xc00ba80b98 pc=0x4907d6 runtime.selectgo(0xc00ba80d68, 0xc00ba80d30, 0x8?, 0x0, 0xc0125ae000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ba80cf8 sp=0xc00ba80bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc008787e00, {0xc015b2f600, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc00ba80d98 sp=0xc00ba80cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc008787e00, {0xc015b2f600?, 0xc00122c030?, 0xc00ba80de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc00ba80dd8 sp=0xc00ba80d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00dc6fe90, {0xc015b2f600?, 0xc00ba80e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc00ba80e20 sp=0xc00ba80dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc00dc6fe90}, {0xc015b2f600, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc00ba80e68 sp=0xc00ba80e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc011e6a360, {0xc015b2f600, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc00ba80ea8 sp=0xc00ba80e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc015b2f5f0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc00ba80f28 sp=0xc00ba80ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x461ce0?, 0xc011e6a360, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc00ba80fc0 sp=0xc00ba80f28 pc=0xc19546 google.golang.org/grpc.recv(0x20?, {0x7f275f301038, 0xc0000af170}, 0x484945?, {0x0?, 0x0?}, {0x51d3160, 0xc008703a40}, 0xc2a566?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc00ba81038 sp=0xc00ba80fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc00844da20, {0x51d3160?, 0xc008703a40}, 0x5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc00ba81140 sp=0xc00ba81038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00ba81170 sp=0xc00ba81140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011e6a120, 0xc00ba81248, 0xc00ba81218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc00ba811c8 sp=0xc00ba81170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011e6a120, {0x51d3160?, 0xc008703a40?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00ba81278 sp=0xc00ba811c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0109102e0, {0x51d3160?, 0xc008703a40?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc00ba812c0 sp=0xc00ba81278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc014688140) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc00ba812f0 sp=0xc00ba812c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc00ba81310 sp=0xc00ba812f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc008787cc0, 0xc00ba81780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc00ba813b8 sp=0xc00ba81310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc006c4ea80?}, {{0xc004c21508, 0x1, 0x8}, {0xc004c21510, 0x1, 0x8}}, {0x17600a17c26c7960, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc00ba819a0 sp=0xc00ba813b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc006c4ea80}, 0xc008787b30, 0xc004c21508?, {{0xc004c21508, 0x1, 0x8}, {0xc004c21510, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc00ba81e80 sp=0xc00ba819a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc006c4ea80?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc00ba81f58 sp=0xc00ba81e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00ba81f78 sp=0xc00ba81f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00ba81fe0 sp=0xc00ba81f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ba81fe8 sp=0xc00ba81fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 404695 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc0151dd640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0151dd5d0 sp=0xc0151dd5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00f500bd0, 0x14b) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0151dd618 sp=0xc0151dd5d0 pc=0x4be30c sync.(*Cond).Wait(0xc005caa400?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0151dd650 sp=0xc0151dd618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc005caaa68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc0151dd708 sp=0xc0151dd650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0055df900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc0151ddcb8 sp=0xc0151dd708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0055df900, {0x624a8a0, 0xc00f500c00}, 0x413a836900000000?, 0xc00b353dc0?, 0xc012510430?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0151ddd70 sp=0xc0151ddcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00f500c00?}, {0xc0059fa000?}, 0x3?, 0xc0041ce1e0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0151dde20 sp=0xc0151ddd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0151ddfe0 sp=0xc0151dde20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0151ddfe8 sp=0xc0151ddfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498308 [select]: runtime.gopark(0xc00a6fffa8?, 0x2?, 0x3?, 0x30?, 0xc00a6fff74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a6ffdc8 sp=0xc00a6ffda8 pc=0x4907d6 runtime.selectgo(0xc00a6fffa8, 0xc00a6fff70, 0xc00dc001e0?, 0x0, 0x1054cca?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a6fff08 sp=0xc00a6ffdc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc00a6fffe0 sp=0xc00a6fff08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a6fffe8 sp=0xc00a6fffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 49080 [select, 7 minutes]: runtime.gopark(0xc007987fb0?, 0x2?, 0x1?, 0x0?, 0xc007987f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bfffe28 sp=0xc00bfffe08 pc=0x4907d6 runtime.selectgo(0xc00bffffb0, 0xc007987f98, 0xc002a40360?, 0x0, 0xc005e241b0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bffff68 sp=0xc00bfffe28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc00bffffe0 sp=0xc00bffff68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bffffe8 sp=0xc00bffffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 397991 [sleep]: runtime.gopark(0x4b8d763b0e1?, 0xc05d1?, 0x1c?, 0x15?, 0x80caee6b7f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004631df0 sp=0xc004631dd0 pc=0x4907d6 time.Sleep(0x14f9d23dc) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004631e30 sp=0xc004631df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00c49d770, 0xc05d1, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004631e88 sp=0xc004631e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00c49d770, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004631ec8 sp=0xc004631e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc004aa0f80?, {0xc011aaa000?, 0x20, 0xc004aa0f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004631fa8 sp=0xc004631ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004631fe0 sp=0xc004631fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004631fe8 sp=0xc004631fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 396803 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x2fb?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011b1af58 sp=0xc011b1af38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539360?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc011b1af90 sp=0xc011b1af58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539278, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc011b1afb0 sp=0xc011b1af90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00cd39480?, 0xc00b101000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc011b1afd8 sp=0xc011b1afb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00cd39480, {0xc00b101000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc011b1b058 sp=0xc011b1afd8 pc=0x535efa net.(*netFD).Read(0xc00cd39480, {0xc00b101000?, 0x7b5a68?, 0xc00cd39480?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc011b1b0a0 sp=0xc011b1b058 pc=0x7a20c9 net.(*conn).Read(0xc002e004e8, {0xc00b101000?, 0x164290f?, 0xc002e004e8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc011b1b0e8 sp=0xc011b1b0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03be72d20, {0xc00b101000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc011b1b130 sp=0xc011b1b0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc003b46af8, {0xc00b101000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc011b1b1d0 sp=0xc011b1b130 pc=0x370f896 bufio.(*Reader).fill(0xc0100d3c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc011b1b208 sp=0xc011b1b1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0100d3c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc011b1b228 sp=0xc011b1b208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0100d4268?, {0x62295c8, 0xc0100d3c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc011b1b260 sp=0xc011b1b228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0100d3c00, 0xc0160a8dd0, 0xc011b1b442, 0xc011b1b444, {0x623b718, 0xc00b6a8240}, 0xc0059fa000, 0xc0034ab0a4, 0xc011b1b440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc011b1b398 sp=0xc011b1b260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0100d3c00, {0x624a948, 0xc00210c5d0}, 0xc0160a8db0, 0xc0059fa000, 0xc00210c1e0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc011b1b768 sp=0xc011b1b398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc00210c5d0}, {0x628a538, 0xc03be72d20}, {{{0xc004c57038, 0x4}}, 0x1, 0xc00210c2a0, 0xc00210c2d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc011b1b8a8 sp=0xc011b1b768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc00210c5d0}, {0x628a538?, 0xc03be72d20}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc011b1be20 sp=0xc011b1b8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00dde9fb0?}, {0x628a538, 0xc03be72d20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc011b1beb0 sp=0xc011b1be20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00dde9fb0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc011b1bf30 sp=0xc011b1beb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011b1bfe0 sp=0xc011b1bf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011b1bfe8 sp=0xc011b1bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498145 [select]: runtime.gopark(0xc007e2be60?, 0x3?, 0x0?, 0x0?, 0xc007e2be32?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007e2bcb8 sp=0xc007e2bc98 pc=0x4907d6 runtime.selectgo(0xc007e2be60, 0xc007e2be2c, 0xc0081f3e30?, 0x0, 0xc005976e18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc007e2bdf8 sp=0xc007e2bcb8 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc010ffa2d0, {0x624a948, 0xc0081f3e30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:383 +0x185 fp=0xc007e2beb0 sp=0xc007e2bdf8 pc=0x19e7805 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop-fm({0x624a948?, 0xc0081f3e30?}) :1 +0x39 fp=0xc007e2bed8 sp=0xc007e2beb0 pc=0x19f8df9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002a40360, {0x624a948, 0xc0081f3e30}, {0x53ddbcd?, 0x6294e00?}, 0xc008ae7fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:324 +0xd1 fp=0xc007e2bf50 sp=0xc007e2bed8 pc=0x1054071 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:338 +0xf9 fp=0xc007e2bfe0 sp=0xc007e2bf50 pc=0x19e75d9 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007e2bfe8 sp=0xc007e2bfe0 pc=0x4c2401 created by time.goFunc GOROOT/src/time/sleep.go:176 +0x32 goroutine 49853 [select, 7 minutes]: runtime.gopark(0xc011969fb0?, 0x2?, 0x64?, 0x0?, 0xc011969f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c247e28 sp=0xc00c247e08 pc=0x4907d6 runtime.selectgo(0xc00c247fb0, 0xc011969f98, 0xc002a40360?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00c247f68 sp=0xc00c247e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc00c247fe0 sp=0xc00c247f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c247fe8 sp=0xc00c247fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 498514 [semacquire]: runtime.gopark(0xae1b769bf2f?, 0xd?, 0x25?, 0x0?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0121fe6f0 sp=0xc0121fe6d0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x34?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc0121fe758 sp=0xc0121fe6f0 pc=0x4a17de sync.runtime_SemacquireMutex(0xc0121fe818?, 0xc8?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc0121fe788 sp=0xc0121fe758 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x7f27439ff820?, 0xc00d3d5980?, 0xc000999310?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc0121fe7e0 sp=0xc0121fe788 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc0034f22c0?, 0xc0121fe870?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc0121fe828 sp=0xc0121fe7e0 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0034f22c0, {0x16?, 0x16?}, 0x0, 0xc015815478, 0xc00568e810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc0121fe928 sp=0xc0121fe828 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc015815200, 0xa8?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc0121fe9c8 sp=0xc0121fe928 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc015815200, {0xc010cb96a8, 0x7, 0x7}, 0xc00238e8a0?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc0121fea78 sp=0xc0121fe9c8 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc00238e8a0?, 0x0?, 0x0?}, {0xc010cb96a8?, 0x7?, 0xc0042f8420?}, 0x30?, 0xff?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc0121feae0 sp=0xc0121fea78 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc00568fde0?, {0xc00238e8a0?, 0x1?, 0x7?}, {0xc010cb96a8?, 0x10?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc0121feb30 sp=0xc0121feae0 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00568fde0, {0xc00238e8a0, 0x7, 0x25}, {0xc010cb96a8, 0x7, 0x7}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc0121feb80 sp=0xc0121feb30 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00568ea20, {0xc010cb96a8?, 0xc007136028?, 0x1000000000000?}, 0x0?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc0121fecb0 sp=0xc0121feb80 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00568ea20, {0xc00238e8a0?, 0x0?, 0x0?}, {0xc010cb96a8?, 0x4a59d2?, 0x7?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc0121fecf0 sp=0xc0121fecb0 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc00568e888, {0xc00238e8a0?, 0x0?, 0x7?}, {0xc010cb96a8, 0x7, 0x7}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc0121fed48 sp=0xc0121fecf0 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00568e500, {0xc010cb96a8, 0x7, 0x7}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc0121fede0 sp=0xc0121fed48 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc007136028, {{0xc0133f8500, 0x6, 0x10}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc0121fee30 sp=0xc0121fede0 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekIntentGE(0xc0116b7000, {0xc0133f8500, 0x6, 0x10}, {0xb9, 0x40, 0xe, 0x54, 0xb3, 0x5b, ...}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:540 +0x117 fp=0xc0121fef58 sp=0xc0121fee30 pc=0x180d357 github.com/cockroachdb/cockroach/pkg/storage.MVCCResolveWriteIntent({0x624a948, 0xc03ea9e8d0}, {0x7f275f2b7158?, 0xc007136000?}, 0x0?, {{{0xc0133f8500, 0x6, 0x10}, {0x0, 0x0, ...}}, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3968 +0x12b fp=0xc0121ff0b0 sp=0xc0121fef58 pc=0x182dfab github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.ResolveIntent({_, _}, {_, _}, {{0x62cc4b0, 0xc00d2dc2e0}, {{0x17600a76e51d432d, 0x0, 0x0}, 0xc010cb9680, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_resolve_intent.go:96 +0x385 fp=0xc0121ff6e8 sp=0xc0121ff0b0 pc=0x1d02d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76e51d432d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc0121fff78 sp=0xc0121ff6e8 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc003bee690, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc012201de8 sp=0xc0121fff78 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc016d7d9a0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc012202438 sp=0xc012201de8 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc016d7d9a0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc012202910 sp=0xc012202438 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a7702eaa82d, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc012203790 sp=0xc012202910 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000580000?, {0x624a948, 0xc03ea9e8d0}, {0xc010cb96a0, 0x8}, 0xc003bee690, 0x8?, 0x0?, {{0x17600a7702eaa82d, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc012204410 sp=0xc012203790 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc03ea9e8d0}, {0xc010cb96a0, 0x8}, 0xc003bee690, 0x400000004?, 0xc012205460, {{0x17600a7702eaa82d, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc0122044c8 sp=0xc012204410 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc006f10000, {0x624a948, 0xc03ea9e8d0}, 0xc003bee690, 0xc016d7d9a0, 0xc012205460, {{0x17600a7702eaa82d, 0x0, 0x0}, {0x17600a76e51d432d, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc012204e90 sp=0xc0122044c8 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc03ea9e8d0}, 0xc003bee690, 0xc016d7d9a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc012205600 sp=0xc012204e90 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc03ea9e8d0}, 0xc003bee690, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc012205920 sp=0xc012205600 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76e51d432d, 0x0, 0x0}, 0xc010cb9680, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc012205ce8 sp=0xc012205920 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e51d432d, 0x0, 0x0}, 0xc010cb9680, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc012206670 sp=0xc012205ce8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x624a948?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0122068b8 sp=0xc012206670 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc03ea9e1e0}, {0xc006a4a780?}, 0xc003bee4b0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc012206f28 sp=0xc0122068b8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc03234bf50}, 0xc003bee4b0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc012206fd0 sp=0xc012206f28 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc03234bf50?}, {0x52f0080?, 0xc003bee4b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc012207000 sp=0xc012206fd0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc03234bf50}, {0x52f0080, 0xc003bee4b0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc012207330 sp=0xc012207000 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03234bf50?}, {0x52f0080?, 0xc003bee4b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc012207370 sp=0xc012207330 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc03234bf50}, {0x52f0080, 0xc003bee4b0}, 0xc00c88d408?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0122073a0 sp=0xc012207370 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03234bf50?}, {0x52f0080?, 0xc003bee4b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0122073e0 sp=0xc0122073a0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc03234bf50?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc012207418 sp=0xc0122073e0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc03234bf50}, {0xc003ca8000?, 0x3?}, 0xc00c88d4d0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc012207490 sp=0xc012207418 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc03234bf50?}, {0x52f0080?, 0xc003bee4b0?}, 0x51b9a20?, 0xc0533f6d80?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc012207508 sp=0xc012207490 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc03234bf50?}, {0x52f0080?, 0xc003bee4b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc012207548 sp=0xc012207508 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc03234bf50?}, {0xc03234bf50?, 0x4?}, {0x52f0080?, 0xc003bee4b0?}, {0x51b9a20?, 0xc0533f6e00}, 0x20300c?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc012207588 sp=0xc012207548 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03234bf50}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003bee4b0}, {0x51b9a20, 0xc0533f6e00}, 0xc00c88d700?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc012207678 sp=0xc012207588 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc03234bf50}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003bee4b0}, {0x51b9a20, 0xc0533f6e00}, 0xc00c88d7e0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc012207710 sp=0xc012207678 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc03234bef0}, 0xc003bee4b0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc012207790 sp=0xc012207710 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc88d830?, {0x624a948?, 0xc03234bef0?}, 0x624a948?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc012207818 sp=0xc012207790 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00e854120, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc012207ab8 sp=0xc012207818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00e854120, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc012207bf0 sp=0xc012207ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc012208338 sp=0xc012207bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc012208c38 sp=0xc012208338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0122094a0 sp=0xc012208c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51bd1a1, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0122098c0 sp=0xc0122094a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0122099f8 sp=0xc0122098c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc03234bb00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc012209c00 sp=0xc0122099f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc03234bb00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc012209f30 sp=0xc012209c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012209fe0 sp=0xc012209f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012209fe8 sp=0xc012209fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498385 [select]: runtime.gopark(0xc01142ec78?, 0x3?, 0x4?, 0x30?, 0xc01142ea9e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01142e900 sp=0xc01142e8e0 pc=0x4907d6 runtime.selectgo(0xc01142ec78, 0xc01142ea98, 0x1?, 0x0, 0xc00aeb3950?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01142ea40 sp=0xc01142e900 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForQuery(0xc004959320, {0x624a948, 0xc00aeb3950}, 0xc03f222c00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:805 +0x585 fp=0xc01142ecb8 sp=0xc01142ea40 pc=0x19bb485 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc00b56cf00, {0x624a948, 0xc00aeb3950}, {0x0, {0x17600a76e5114beb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:358 +0x175 fp=0xc01142ecf8 sp=0xc01142ecb8 pc=0x1cbd995 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc00aeb3950}, 0xc003024960, 0xc01142f298?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc01142f168 sp=0xc01142ecf8 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc00aeb3950}, 0xc01142f350?, {0x0, {0x17600a76e5114beb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc01142f2a8 sp=0xc01142f168 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc00aeb3950}, 0xc017227d10, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc01142f5c8 sp=0xc01142f2a8 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc01142f990 sp=0xc01142f5c8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc011430318 sp=0xc01142f990 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc011430560 sp=0xc011430318 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc00aeb38f0}, {0xc001784510?}, 0xc017227c20) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc011430bd0 sp=0xc011430560 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc00aeb3830}, 0xc017227c20) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc011430c78 sp=0xc011430bd0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc00aeb3830?}, {0x52f0080?, 0xc017227c20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc011430ca8 sp=0xc011430c78 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc00aeb3830}, {0x52f0080, 0xc017227c20}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc011430fd8 sp=0xc011430ca8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00aeb3830?}, {0x52f0080?, 0xc017227c20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc011431018 sp=0xc011430fd8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc00aeb3830}, {0x52f0080, 0xc017227c20}, 0xc0114310b0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc011431048 sp=0xc011431018 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00aeb3830?}, {0x52f0080?, 0xc017227c20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc011431088 sp=0xc011431048 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc00aeb3830?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0114310c0 sp=0xc011431088 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00aeb3830}, {0x0?, 0x0?}, 0xc011431178) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc011431138 sp=0xc0114310c0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc00aeb3830?}, {0x52f0080?, 0xc017227c20?}, 0x62235e0?, 0x52a4f5?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0114311b0 sp=0xc011431138 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc00aeb3830?}, {0x52f0080?, 0xc017227c20?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0114311f0 sp=0xc0114311b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc00aeb3830?}, {0xc00aeb3830?, 0x4?}, {0x52f0080?, 0xc017227c20?}, {0x51b9a20?, 0xc0533f6b80}, 0x203002?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc011431230 sp=0xc0114311f0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb3830}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227c20}, {0x51b9a20, 0xc0533f6b80}, 0xc0114313a8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc011431320 sp=0xc011431230 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc00aeb3830}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227c20}, {0x51b9a20, 0xc0533f6b80}, 0xc011431488?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc0114313b8 sp=0xc011431320 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc00aeb37d0}, 0xc017227c20, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc011431438 sp=0xc0114313b8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc002698400?, {0x624a948?, 0xc00aeb37d0?}, 0xc00c88d4a8?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc0114314c0 sp=0xc011431438 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0079ff200, {_, _}, _, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc011431760 sp=0xc0114314c0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0079ff200, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc011431898 sp=0xc011431760 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc011431fe0 sp=0xc011431898 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0114328e0 sp=0xc011431fe0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc011433148 sp=0xc0114328e0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x17600a76e511605f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc011433568 sp=0xc011433148 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0114336a0 sp=0xc011433568 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc0114337c8 sp=0xc0114336a0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0x4b6f420?, {_, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc0114338e8 sp=0xc0114337c8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e5114beb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc0114339f8 sp=0xc0114338e8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc011e015f0}, 0x0?, 0xc017500100) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc011433bf0 sp=0xc0114339f8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc011e015f0}, 0x4596bd?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc011433c38 sp=0xc011433bf0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc004959320, {0x624a948, 0xc011e015f0}, {{0x93, 0xf, 0x9f, 0x6f, 0xe0, 0x86, 0x43, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc011433c88 sp=0xc011433c38 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).startQueryPusherTxn.func1({0x624a948, 0xc011e015f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:854 +0x238 fp=0xc011433f30 sp=0xc011433c88 pc=0x19bbe78 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011433fe0 sp=0xc011433f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011433fe8 sp=0xc011433fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 479352 [sleep]: runtime.gopark(0x4b890f5d454?, 0x8019aa?, 0xf8?, 0x3e?, 0x806f8c0155?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0039c9df0 sp=0xc0039c9dd0 pc=0x4907d6 time.Sleep(0x13aa8975f) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc0039c9e30 sp=0xc0039c9df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc01cd04840, 0x8019aa, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc0039c9e88 sp=0xc0039c9e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc01cd04840, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc0039c9ec8 sp=0xc0039c9e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc004fb6000?, 0x2c, 0xc001405fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc0039c9fa8 sp=0xc0039c9ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc0039c9fe0 sp=0xc0039c9fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0039c9fe8 sp=0xc0039c9fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 115791 [semacquire, 6 minutes]: runtime.gopark(0x18?, 0xc003440400?, 0x80?, 0xad?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00168daa8 sp=0xc00168da88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc006c4ead0, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00168db10 sp=0xc00168daa8 pc=0x4a17de sync.runtime_Semacquire(0xc006c4eac0?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00168db40 sp=0xc00168db10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00168db68 sp=0xc00168db40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc006c4eac0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc00168db80 sp=0xc00168db68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc006c4eac0?, {0x624a8a0?, 0xc006c4ea80?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc00168dbb0 sp=0xc00168db80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc006c4e9c0}, {0xc006c4ea40, 0x1, 0xc00376dec0?}, 0x0, 0xc00376de60, {0xc0070554b0, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc00168dd38 sp=0xc00168dbb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00157bed0?, {0x624a8a0, 0xc006c4e9c0}, {0xc010d81ef0, 0x1, 0x8c0?}, {0xc010875b00?, 0x9c14400?, 0x0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc00168de58 sp=0xc00168dd38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc00157bf30?, {0x624a8a0?, 0xc006c4e9c0?}, {0xc010d81ef0?, 0x1a01418?, 0x106663b80?}, {0xc007e02190?, 0x157bf50?, 0xc0?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc00168ded0 sp=0xc00168de58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc006c4e9c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc00168df58 sp=0xc00168ded0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00168df78 sp=0xc00168df58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00168dfe0 sp=0xc00168df78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00168dfe8 sp=0xc00168dfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498422 [runnable]: runtime.gopark(0xae1b6f4af5e?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc008962f70 sp=0xc008962f50 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee7fc, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc008962fd8 sp=0xc008962f70 pc=0x4a17de sync.runtime_SemacquireMutex(0x0?, 0x0?, 0x0?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc008963008 sp=0xc008962fd8 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee7e0, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc008963060 sp=0xc008963008 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc0089630a8 sp=0xc008963060 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0107fc580, {0xc0089631e8?, 0x4ccece?}, 0x0, 0x0, 0xc014074810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc0089631a8 sp=0xc0089630a8 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00c730900, 0xc0107fc580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc0089631f8 sp=0xc0089631a8 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc0107fc580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc014074810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc008963280 sp=0xc0089631f8 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc0029416b0, 0x0?, 0xc014075e30, {0x0?, 0xc014074810?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc0089633b0 sp=0xc008963280 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x0?, 0x0?, 0x0?, {0x0?, 0xc014074810?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc008963400 sp=0xc0089633b0 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc014075de0?, 0xc0069086e0?, {0x0?, 0xc014074810?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc008963448 sp=0xc008963400 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc014075de0, 0xc0069086e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc0089634a8 sp=0xc008963448 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc014075de0, {0xc0074954c0, 0x20, 0x40}, {0xc006a8a460, 0x20, 0x20}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc0089634f8 sp=0xc0089634a8 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc014074a20, {0xc006a8a460?, 0x1072c02?, 0x100c00641d368?}, 0x1f?, 0x1f?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc008963628 sp=0xc0089634f8 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc014074a20, {0xc0074954c0?, 0x0?, 0x20?}, {0xc006a8a460?, 0x4a59d2?, 0x20?}, 0x80?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc008963668 sp=0xc008963628 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc014074888, {0xc0074954c0?, 0x40?, 0xc006a8a460?}, {0xc006a8a460, 0x20, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc0089636c0 sp=0xc008963668 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc014074500, {0xc006a8a460, 0x20, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc008963758 sp=0xc0089636c0 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc007136c28, {{0xc0150cc060, 0x1f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc0089637a8 sp=0xc008963758 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc00dd4d200, {0x624a948, 0xc008364d20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc008963810 sp=0xc0089637a8 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc008364d20}, {0x62b4fe0, 0xc007136c28}, {0xc0150cc060, 0x1f, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc008963de0 sp=0xc008963810 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc008364d20}, {0x628f6b0?, 0xc007136c00?}, {0xc0150cc060, 0x1f, 0x30}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc008963f60 sp=0xc008963de0 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc008364d20?}, {0x628f6b0?, 0xc007136c00?}, {0xc0150cc060?, 0x19?, 0x1d?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc008964050 sp=0xc008963f60 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.EndTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc00dc76fc0}, {{0x17600a765c4f6273, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:238 +0x2fa fp=0xc0089656d0 sp=0xc008964050 pc=0x1ce7a1a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a765c4f6273, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc008965f60 sp=0xc0089656d0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc003dffa40, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc008967dd0 sp=0xc008965f60 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc006598a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc008968420 sp=0xc008967dd0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc006598a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc0089688f8 sp=0xc008968420 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a767a1cc773, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc008969778 sp=0xc0089688f8 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc002100400?, {0x624a948, 0xc008364d20}, {0xc0034aa400, 0x8}, 0xc003dffa40, 0x8?, 0x0?, {{0x17600a767a1cc773, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00896a3f8 sp=0xc008969778 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc008364d20}, {0xc0034aa400, 0x8}, 0xc003dffa40, 0x400000004?, 0xc00896b448, {{0x17600a767a1cc773, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00896a4b0 sp=0xc00896a3f8 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc006f10000, {0x624a948, 0xc008364d20}, 0xc003dffa40, 0xc006598a00, 0xc00896b448, {{0x17600a767a1cc773, 0x0, 0x0}, {0x17600a765c4f6273, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00896ae78 sp=0xc00896a4b0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc008364d20}, 0xc003dffa40, 0xc006598a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00896b5e8 sp=0xc00896ae78 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc008364d20}, 0xc003dffa40, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00896b908 sp=0xc00896b5e8 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a765c4f6273, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00896bcd0 sp=0xc00896b908 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a765c4f6273, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00896c658 sp=0xc00896bcd0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x3?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00896c8a0 sp=0xc00896c658 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc008364c90}, {0xc00a1cf500?}, 0xc003dff860) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00896cf10 sp=0xc00896c8a0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc008364ae0}, 0xc003dff860) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00896cfb8 sp=0xc00896cf10 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc008364ae0?}, {0x52f0080?, 0xc003dff860?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00896cfe8 sp=0xc00896cfb8 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc008364ae0}, {0x52f0080, 0xc003dff860}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00896d318 sp=0xc00896cfe8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc008364ae0?}, {0x52f0080?, 0xc003dff860?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00896d358 sp=0xc00896d318 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc008364ae0}, {0x52f0080, 0xc003dff860}, 0xc00d1ad3f0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00896d388 sp=0xc00896d358 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc008364ae0?}, {0x52f0080?, 0xc003dff860?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00896d3c8 sp=0xc00896d388 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc008364ae0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00896d400 sp=0xc00896d3c8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc008364ae0}, {0xc002a40360?, 0x624a948?}, 0xc00d1ad4b8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00896d478 sp=0xc00896d400 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc008364ae0?}, {0x52f0080?, 0xc003dff860?}, 0xc0134d2ba0?, 0x52f0080?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00896d4f0 sp=0xc00896d478 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc008364ae0?}, {0x52f0080?, 0xc003dff860?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00896d530 sp=0xc00896d4f0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc008364ae0?}, {0xc008364ae0?, 0x4?}, {0x52f0080?, 0xc003dff860?}, {0x51b9a20?, 0xc011794380}, 0x203002?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00896d570 sp=0xc00896d530 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc008364ae0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dff860}, {0x51b9a20, 0xc011794380}, 0xc00d1ad6e8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00896d660 sp=0xc00896d570 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc008364ae0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003dff860}, {0x51b9a20, 0xc011794380}, 0xc00d1ad7c8?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00896d6f8 sp=0xc00896d660 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc0083649f0}, 0xc003dff860, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00896d778 sp=0xc00896d6f8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x4?, {0x624a948?, 0xc0083649f0?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00896d800 sp=0xc00896d778 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc015f0ed80, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00896daa0 sp=0xc00896d800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc015f0ed80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00896dbd8 sp=0xc00896daa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00896e320 sp=0xc00896dbd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00896ec20 sp=0xc00896e320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00896f488 sp=0xc00896ec20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e37043a2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00896f8a8 sp=0xc00896f488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0225a20b8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00896fb00 sp=0xc00896f8a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0225a2080, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00896fc28 sp=0xc00896fb00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc016171170}, {0x7f275f30db68, 0xc0225a2080}, 0xc008268fc0, {0xc007dea000, 0x1e, 0x1f}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc00896fe60 sp=0xc00896fc28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc016171170}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc00896ff30 sp=0xc00896fe60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00896ffe0 sp=0xc00896ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00896ffe8 sp=0xc00896ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 480358 [sleep]: runtime.gopark(0x4b90b6b8a7e?, 0x801967?, 0xa5?, 0x40?, 0x80f3069704?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00d017df0 sp=0xc00d017dd0 pc=0x4907d6 time.Sleep(0x143ad7696) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00d017e30 sp=0xc00d017df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc016184c00, 0x801967, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00d017e88 sp=0xc00d017e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc016184c00, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00d017ec8 sp=0xc00d017e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc00f2d7900?, 0x10, 0xc0039d9fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00d017fa8 sp=0xc00d017ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00d017fe0 sp=0xc00d017fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00d017fe8 sp=0xc00d017fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498377 [select]: runtime.gopark(0xc008539c18?, 0x3?, 0x0?, 0x85?, 0xc008539b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0085398f0 sp=0xc0085398d0 pc=0x4907d6 runtime.selectgo(0xc008539c18, 0xc008539b34, 0xc00e3eb8c0?, 0x0, 0xa?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc008539a30 sp=0xc0085398f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc0134b5e60}, {0x6221560, 0xc0061c6000}, {0x46?, {0xc008539e01?, 0x8539e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc008539da0 sp=0xc008539a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc0134b5e60?}, 0x16e, 0xc0061c6000, 0x1a?, 0xc0189748c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc008539e70 sp=0xc008539da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc0134b5e60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc008539f30 sp=0xc008539e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc008539fe0 sp=0xc008539f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc008539fe8 sp=0xc008539fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 404931 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc015e65640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc015e655d0 sp=0xc015e655b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00c845950, 0x18b) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc015e65618 sp=0xc015e655d0 pc=0x4be30c sync.(*Cond).Wait(0xc003491c00?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc015e65650 sp=0xc015e65618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc003492268) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc015e65708 sp=0xc015e65650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0040dc000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc015e65cb8 sp=0xc015e65708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0040dc000, {0x624a8a0, 0xc00c845980}, 0x41157d6c00000000?, 0xc0044e1dc0?, 0xc0172dc750?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc015e65d70 sp=0xc015e65cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00c845980?}, {0xc0059fa000?}, 0x3?, 0xc009f92300?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc015e65e20 sp=0xc015e65d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc015e65fe0 sp=0xc015e65e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc015e65fe8 sp=0xc015e65fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 45302 [IO wait]: runtime.gopark(0xc00608cd80?, 0xb?, 0x0?, 0x0?, 0x58?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00608ca80 sp=0xc00608ca60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x42?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00608cab8 sp=0xc00608ca80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd348, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00608cad8 sp=0xc00608cab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc002cccb00?, 0xc00e1ea000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00608cb00 sp=0xc00608cad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc002cccb00, {0xc00e1ea000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00608cb80 sp=0xc00608cb00 pc=0x535efa net.(*netFD).Read(0xc002cccb00, {0xc00e1ea000?, 0x50c7cc0?, 0x1?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00608cbc8 sp=0xc00608cb80 pc=0x7a20c9 net.(*conn).Read(0xc00de7c168, {0xc00e1ea000?, 0xc02951f500?, 0xc02951f6e0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00608cc10 sp=0xc00608cbc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc007cfecc0, {0xc007f9a040, 0x9, 0xc00cf3533c?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc00608cc48 sp=0xc00608cc10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc007cfecc0}, {0xc007f9a040, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc00608cc90 sp=0xc00608cc48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc007f9a040?, 0x9?, 0xc02951f5c0?}, {0x62052c0?, 0xc007cfecc0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc00608cce0 sp=0xc00608cc90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc007f9a000) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc00608cd90 sp=0xc00608cce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00db2d4a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc00608cfc8 sp=0xc00608cd90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc00608cfe0 sp=0xc00608cfc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00608cfe8 sp=0xc00608cfe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 490512 [select]: runtime.gopark(0xc00150ffa8?, 0x2?, 0x3?, 0x30?, 0xc00150ff74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00150fdc8 sp=0xc00150fda8 pc=0x4907d6 runtime.selectgo(0xc00150ffa8, 0xc00150ff70, 0x9bf9280?, 0x0, 0x7fb6a9690e?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00150ff08 sp=0xc00150fdc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc00150ffe0 sp=0xc00150ff08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00150ffe8 sp=0xc00150ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 339454 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x1c3?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0101a4f58 sp=0xc0101a4f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539540?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0101a4f90 sp=0xc0101a4f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539458, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0101a4fb0 sp=0xc0101a4f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc011e4ce80?, 0xc0054ff000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0101a4fd8 sp=0xc0101a4fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011e4ce80, {0xc0054ff000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0101a5058 sp=0xc0101a4fd8 pc=0x535efa net.(*netFD).Read(0xc011e4ce80, {0xc0054ff000?, 0x7b5a68?, 0xc011e4ce80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0101a50a0 sp=0xc0101a5058 pc=0x7a20c9 net.(*conn).Read(0xc014e5b0b8, {0xc0054ff000?, 0x164290f?, 0xc014e5b0b8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0101a50e8 sp=0xc0101a50a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0134e6240, {0xc0054ff000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc0101a5130 sp=0xc0101a50e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc059d131a0, {0xc0054ff000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc0101a51d0 sp=0xc0101a5130 pc=0x370f896 bufio.(*Reader).fill(0xc0034bb090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc0101a5208 sp=0xc0101a51d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc0034bb090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc0101a5228 sp=0xc0101a5208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0034bb668?, {0x62295c8, 0xc0034bb090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc0101a5260 sp=0xc0101a5228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0034bb000, 0xc003451a90, 0xc0101a5442, 0xc0101a5444, {0x623b718, 0xc0112a4a20}, 0xc0059fa000, 0xc00d63b5c4, 0xc0101a5440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc0101a5398 sp=0xc0101a5260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0034bb000, {0x624a948, 0xc0133a9620}, 0xc003451a80, 0xc0059fa000, 0xc0133a9560, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc0101a5768 sp=0xc0101a5398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0133a9620}, {0x628a538, 0xc0134e6240}, {{{0xc002cb0038, 0x4}}, 0x1, 0xc0133a9590, 0xc0133a95c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc0101a58a8 sp=0xc0101a5768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0133a9620}, {0x628a538?, 0xc0134e6240}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc0101a5e20 sp=0xc0101a58a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0133a9380?}, {0x628a538, 0xc0134e6240}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc0101a5eb0 sp=0xc0101a5e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0133a9380}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc0101a5f30 sp=0xc0101a5eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0101a5fe0 sp=0xc0101a5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0101a5fe8 sp=0xc0101a5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498425 [select]: runtime.gopark(0xc00f6631f0?, 0x2?, 0xc8?, 0x44?, 0xc00f6631ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f663078 sp=0xc00f663058 pc=0x4907d6 runtime.selectgo(0xc00f6631f0, 0xc00f6631e8, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f6631b8 sp=0xc00f663078 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0109a27e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00f663220 sp=0xc00f6631b8 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc010e0e000, {0x51b9a20?, 0xc047820280}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00f663328 sp=0xc00f663220 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x101?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00f663358 sp=0xc00f663328 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0109a2360, 0xc00f663430, 0xc00f663400) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00f6633b0 sp=0xc00f663358 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0109a2360, {0x51b9a20?, 0xc047820280?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00f663460 sp=0xc00f6633b0 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00e1c5ce0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc00ecf5680}, {0x51b9a20, 0xc047820280}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00f6634c8 sp=0xc00f663460 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00e1c5ce0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00ecf5680}, {0x51b9a20, 0xc047820280}, 0xf0de949138776be?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00f6635b8 sp=0xc00f6634c8 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x4427dc5c298f28c0?, {0x624a948?, 0xc00e1c5ce0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc00ecf5680?}, {0x51b9a20?, 0xc047820280?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00f663658 sp=0xc00f6635b8 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123bf490, {0x624a948, 0xc00e1c5ce0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00f6636d8 sp=0xc00f663658 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123bf490?}}, {0x624a948, 0xc00e1c5ce0}, 0xc00ecf5680, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00f6637b0 sp=0xc00f6636d8 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xd1ad818?, {0x624a948?, 0xc00e1c5ce0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00d1ad910?}) :1 +0x65 fp=0xc00f663800 sp=0xc00f6637b0 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0152e8f60, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f663aa0 sp=0xc00f663800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0152e8f60, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3cbda3b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f663bd8 sp=0xc00f663aa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3cbda3b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f664320 sp=0xc00f663bd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3cbda3b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f664c20 sp=0xc00f664320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3cbda3b, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f665488 sp=0xc00f664c20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3cbda3b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f6658a8 sp=0xc00f665488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00b50bc38, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00f665b00 sp=0xc00f6658a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00b50bc00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00f665c28 sp=0xc00f665b00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc054b7b050}, {0x7f275f30db68, 0xc00b50bc00}, 0xc03f543440, {0xc006eeb440, 0x2, 0x4}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc00f665e60 sp=0xc00f665c28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc054b7b050}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc00f665f30 sp=0xc00f665e60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f665fe0 sp=0xc00f665f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f665fe8 sp=0xc00f665fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 492441 [sleep]: runtime.gopark(0x4b848fb5cfd?, 0x2005654?, 0x59?, 0xd6?, 0x80091f04af?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001577df0 sp=0xc001577dd0 pc=0x4907d6 time.Sleep(0x11c36149b) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc001577e30 sp=0xc001577df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00e48d200, 0x2005654, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc001577e88 sp=0xc001577e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00e48d200, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc001577ec8 sp=0xc001577e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc007c8a500?, 0xa, 0xc003fabfb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc001577fa8 sp=0xc001577ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc001577fe0 sp=0xc001577fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001577fe8 sp=0xc001577fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 488387 [chan receive]: runtime.gopark(0x1?, 0x2?, 0x0?, 0x0?, 0xc00296b6c0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00157ce30 sp=0xc00157ce10 pc=0x4907d6 runtime.chanrecv(0xc0032acc00, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc00157cec0 sp=0xc00157ce30 pc=0x45a91b runtime.chanrecv1(0xc00157cf48?, 0xc00157cf20?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc00157cee8 sp=0xc00157cec0 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Wait(0xc0100cae00) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:506 +0x191 fp=0xc00157cf78 sp=0xc00157cee8 pc=0x25bf391 github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).runFlowNow.func2() github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:204 +0x4b fp=0xc00157cfe0 sp=0xc00157cf78 pc=0x25c2fcb runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00157cfe8 sp=0xc00157cfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).runFlowNow github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:203 +0x4af goroutine 498497 [select]: runtime.gopark(0xc00ba21208?, 0x2?, 0x80?, 0xaa?, 0xc00ba21204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ba21090 sp=0xc00ba21070 pc=0x4907d6 runtime.selectgo(0xc00ba21208, 0xc00ba21200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ba211d0 sp=0xc00ba21090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc02d8890e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00ba21238 sp=0xc00ba211d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030826a50, {0x51b9a20?, 0xc0533f6700}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00ba21340 sp=0xc00ba21238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00ba21370 sp=0xc00ba21340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc02d888c60, 0xc00ba21448, 0xc00ba21418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00ba213c8 sp=0xc00ba21370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc02d888c60, {0x51b9a20?, 0xc0533f6700?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00ba21478 sp=0xc00ba213c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00aeb25d0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc017227680}, {0x51b9a20, 0xc0533f6700}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00ba214e0 sp=0xc00ba21478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb25d0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227680}, {0x51b9a20, 0xc0533f6700}, 0xf5a78026e1ae1ab?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00ba215d0 sp=0xc00ba214e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00aeb25d0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc017227680?}, {0x51b9a20?, 0xc0533f6700?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00ba21670 sp=0xc00ba215d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277440, {0x624a948, 0xc00aeb25d0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00ba216f0 sp=0xc00ba21670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277440?}}, {0x624a948, 0xc00aeb25d0}, 0xc017227680, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00ba217c8 sp=0xc00ba216f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00aeb25d0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc00ba21818 sp=0xc00ba217c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc011d63ce0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00ba21ab8 sp=0xc00ba21818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc011d63ce0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4a03826, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00ba21bf0 sp=0xc00ba21ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4a03826, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00ba22338 sp=0xc00ba21bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4a03826, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00ba22c38 sp=0xc00ba22338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4a03826, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00ba234a0 sp=0xc00ba22c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e4a03826, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00ba238c0 sp=0xc00ba234a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00ba239f8 sp=0xc00ba238c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00aeb25a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00ba23c00 sp=0xc00ba239f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00aeb25a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00ba23f30 sp=0xc00ba23c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00ba23fe0 sp=0xc00ba23f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00ba23fe8 sp=0xc00ba23fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 48620 [select, 7 minutes]: runtime.gopark(0xc003673f98?, 0x2?, 0x0?, 0x0?, 0xc003673f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0106cce00 sp=0xc0106ccde0 pc=0x4907d6 runtime.selectgo(0xc0106ccf98, 0xc003673f78, 0xc015467a80?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0106ccf40 sp=0xc0106cce00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc015b24fc0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc0106ccfc8 sp=0xc0106ccf40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc0106ccfe0 sp=0xc0106ccfc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0106ccfe8 sp=0xc0106ccfe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 68245 [select]: runtime.gopark(0xc0186f0160?, 0x2?, 0x80?, 0xb3?, 0xc0186f015c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0186effe8 sp=0xc0186effc8 pc=0x4907d6 runtime.selectgo(0xc0186f0160, 0xc0186f0158, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0186f0128 sp=0xc0186effe8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc006976fc0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0186f0190 sp=0xc0186f0128 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0308260b0, {0x51b9a20?, 0xc003165980}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0186f0298 sp=0xc0186f0190 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xed?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0186f02c8 sp=0xc0186f0298 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc006976d80, 0xc0186f03a0, 0xc0186f0370) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0186f0320 sp=0xc0186f02c8 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc006976d80, {0x51b9a20?, 0xc003165980?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0186f03d0 sp=0xc0186f0320 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc0153576e0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0172262d0}, {0x51b9a20, 0xc003165980}, 0x0?, {0xc015b24f40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0186f0438 sp=0xc0186f03d0 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0153576e0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0172262d0}, {0x51b9a20, 0xc003165980}, 0x80754976d325310?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0186f0528 sp=0xc0186f0438 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x7e3d8b96002f0b9b?, {0x624a948?, 0xc0153576e0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0172262d0?}, {0x51b9a20?, 0xc003165980?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc0186f05c8 sp=0xc0186f0528 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277198, {0x624a948, 0xc0153576e0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0186f0648 sp=0xc0186f05c8 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277198?}}, {0x624a948, 0xc0153576e0}, 0xc0172262d0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0186f0720 sp=0xc0186f0648 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x21fa4788?, {0x624a948?, 0xc0153576e0?}, 0x624a948?, {0x0?, 0xc006a4afb0?, 0xc021fa4880?}) :1 +0x65 fp=0xc0186f0770 sp=0xc0186f0720 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0107c9980, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0186f0a10 sp=0xc0186f0770 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0107c9980, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0186f0b48 sp=0xc0186f0a10 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0186f1290 sp=0xc0186f0b48 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0186f1b90 sp=0xc0186f1290 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1403 +0x1393 fp=0xc0186f23f8 sp=0xc0186f1b90 pc=0x19c8fb3 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3ffad23, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0186f2818 sp=0xc0186f23f8 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc024bdf138, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc0186f2a70 sp=0xc0186f2818 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc024bdf100, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc0186f2b98 sp=0xc0186f2a70 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc024bdf0d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc0186f2e28 sp=0xc0186f2b98 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc024bdefd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc0186f33c8 sp=0xc0186f2e28 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc024bdefd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc0186f36b8 sp=0xc0186f33c8 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc024bdeea0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc0186f39b8 sp=0xc0186f36b8 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc024bdee80?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc0186f3b60 sp=0xc0186f39b8 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc024bdedd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc0186f3d78 sp=0xc0186f3b60 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc024bdec00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc0186f3fd0 sp=0xc0186f3d78 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc0186f40f8 sp=0xc0186f3fd0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00844d810, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc0186f43c0 sp=0xc0186f40f8 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc0186f44d8 sp=0xc0186f43c0 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00055dc80, {0x624a948, 0xc0153575f0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc0186f46f8 sp=0xc0186f44d8 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00055dc80, {0x624a948, 0xc0153575f0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc0186f49f0 sp=0xc0186f46f8 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc0077ae480, {0x624a948, 0xc0153575f0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc0186f4ad0 sp=0xc0186f49f0 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc000e81500, {0x624a948, 0xc0153575f0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:629 +0x8a fp=0xc0186f4da8 sp=0xc0186f4ad0 pc=0x320072a github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc00fe614a0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x30 fp=0xc0186f4e00 sp=0xc0186f4da8 pc=0x3204f70 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc012bd1480) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 fp=0xc0186f4e18 sp=0xc0186f4e00 pc=0x2860090 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc008738620) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:202 +0x2e fp=0xc0186f4e68 sp=0xc0186f4e18 pc=0x2860bee github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:48 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc00a45ef70) github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:195 +0x4a fp=0xc0186f4fc8 sp=0xc0186f4e68 pc=0x299be6a github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0077ae840) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f fp=0xc0186f5088 sp=0xc0186f4fc8 pc=0x293bf5f github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc003552f00) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286 +0x73 fp=0xc0186f50b0 sp=0xc0186f5088 pc=0x29a0db3 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc003552f00) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:311 +0x1e fp=0xc0186f50c8 sp=0xc0186f50b0 pc=0x29a0f5e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter-fm() :1 +0x26 fp=0xc0186f50e0 sp=0xc0186f50c8 pc=0x2a274e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0038d0800?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc0186f5120 sp=0xc0186f50e0 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc003552f00) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:317 +0x4c fp=0xc0186f5158 sp=0xc0186f5120 pc=0x29a100c github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc005e02300, {{0xc0038d0800?, 0xc00483a578?}, 0x624cee8?, 0xc021fa9220?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc0186f51d0 sp=0xc0186f5158 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc01ae5ce00, {{0x624a948, 0xc015357440}, 0xc0038d0800, 0xc00483a578}) github.com/cockroachdb/cockroach/pkg/sql/update.go:164 +0xc4 fp=0xc0186f5230 sp=0xc0186f51d0 pc=0x352d844 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc01052ae10, {{0x624a948?, 0xc015357440?}, 0xc0038d0800?, 0xc00483a578?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc0186f5288 sp=0xc0186f5230 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc021fa9488?, 0x624d578?}, {0x624ceb0, 0xc01052ae10}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc0186f5338 sp=0xc0186f5288 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc0186f5370 sp=0xc0186f5338 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc021fa9488, {0x624ceb0?, 0xc01052ae10?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc0186f5400 sp=0xc0186f5370 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc021fa9488, {0x624ceb0, 0xc01052ae10}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc0186f5450 sp=0xc0186f5400 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc015357440?}, 0xc0038d0800?, 0xc00483a578?}, {0x624ceb0, 0xc01052ae10}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc0186f54d0 sp=0xc0186f5450 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc010a7bb80, {0x624a948, 0xc0153573e0}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc0186f5510 sp=0xc0186f54d0 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:122 +0x34 fp=0xc0186f5538 sp=0xc0186f5510 pc=0x3224ef4 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc013dee7e0?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc0186f5578 sp=0xc0186f5538 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc013dee7e0, {0x624a8a0?, 0xc012bd1440?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:121 +0x70 fp=0xc0186f55d0 sp=0xc0186f5578 pc=0x3224e70 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc013dee7e0, {0x624a8a0?, 0xc012bd1440?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:728 +0x3a fp=0xc0186f5610 sp=0xc0186f55d0 pc=0x230ec1a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00596e800, {0x624a8a0, 0xc012bd1440}, 0xc013dee7e0?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc0186f56d0 sp=0xc0186f5610 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc012bd1440?}, 0xc00844d810?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc0186f5770 sp=0xc0186f56d0 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc015356540}, 0xc00dcede00, 0xc00844d810, 0xc003165780, 0xc0046ab500, 0xc00483a8b0, 0xc015333eb0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc0186f6020 sp=0xc0186f5770 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x0?, {0x624a948?, 0xc015356540}, 0xe8?, 0xc00dcede00, 0x83608a91094c8d01?, {{0x624ceb0?, 0xc01052ae10?}, 0x0?}, 0xc0046ab500, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc0186f60e0 sp=0xc0186f6020 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0052d77c0?, {0x624a948, 0xc015356540}, 0xc00483a8b0, 0xc00483a578?, 0xc00483a578, 0xc0046ab500, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc0186f61f8 sp=0xc0186f60e0 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00483a000, {0x624a948?, 0xc015356540}, 0xc00483a578, 0xc015356540?, {0x7f275f329680?, 0xc0100d5ac8}, 0x6e?, 0xc008738368) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc0186f6300 sp=0xc0186f61f8 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00483a000, {0x624a8a0, 0xc012bd0780}, 0xc00483a578, {0x7f275f329680, 0xc0100d5ac8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc0186f6720 sp=0xc0186f6300 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00483a000, {0x624a948?, 0xc00ddf30b0?}, {{0x6274800, 0xc0134b1500}, {0xc00a4e4005, 0x6e}, 0x3, 0x1}, 0xc014f0f200, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc0186f70b0 sp=0xc0186f6720 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc00ddf30b0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc0186f7168 sp=0xc0186f70b0 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc00ddf30b0?}, {0x6274800?, 0xc0134b1500?}, 0xc00a4e4000?, 0x203003?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc0186f72b8 sp=0xc0186f7168 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00483a000, {0x624a948, 0xc00ddf30b0}, {{0x6274800, 0xc0134b1500}, {0xc00a4e4005, 0x6e}, 0x3, 0x1}, 0xc014f0f200, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc0186f7430 sp=0xc0186f72b8 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00483a000, {0x624a948, 0xc00ddf30b0}, {0xc014f0f200, {0xc00ddf3170, 0x3, 0x3}, {0x9c5f2e8, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:220 +0x173 fp=0xc0186f7550 sp=0xc0186f7430 pc=0x3307493 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc005ecc28e, 0x0}, 0x0, {0xc1116e8196db0041, 0x7fecca2e91, 0x0}, 0x1}, 0xc00483a000, 0xc0186f78c0, 0xc0186f78b0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2201 +0x565 fp=0xc0186f7708 sp=0xc0186f7550 pc=0x32fafc5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00483a000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2203 +0x6d3 fp=0xc0186f7cb8 sp=0xc0186f7708 pc=0x32f9153 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00483a000, {0x624a8a0, 0xc00e678ec0}, 0x41175ae000000000?, 0xc00cf59dc0?, 0xc000763380?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0186f7d70 sp=0xc0186f7cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00e678ec0?}, {0xc0059fa000?}, 0x3?, 0xc006defed8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0186f7e20 sp=0xc0186f7d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0186f7fe0 sp=0xc0186f7e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0186f7fe8 sp=0xc0186f7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498368 [select]: runtime.gopark(0xc0085cfc18?, 0x3?, 0xd2?, 0xf9?, 0xc0085cfb3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0085cf8f0 sp=0xc0085cf8d0 pc=0x4907d6 runtime.selectgo(0xc0085cfc18, 0xc0085cfb34, 0xc00a1d6700?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0085cfa30 sp=0xc0085cf8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc012b41350}, {0x6221560, 0xc00d642240}, {0x46?, {0xc0085cfe01?, 0x85cfe60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc0085cfda0 sp=0xc0085cfa30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc012b41350?}, 0x164, 0xc00d642240, 0x40?, 0xc00d57ed20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc0085cfe70 sp=0xc0085cfda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc012b41350}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc0085cff30 sp=0xc0085cfe70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0085cffe0 sp=0xc0085cff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0085cffe8 sp=0xc0085cffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 118155 [IO wait]: runtime.gopark(0xc0049f0d80?, 0xb?, 0x0?, 0x0?, 0x110?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0049f0a80 sp=0xc0049f0a60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x41?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0049f0ab8 sp=0xc0049f0a80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd078, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0049f0ad8 sp=0xc0049f0ab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc011b04880?, 0xc010914000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0049f0b00 sp=0xc0049f0ad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011b04880, {0xc010914000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0049f0b80 sp=0xc0049f0b00 pc=0x535efa net.(*netFD).Read(0xc011b04880, {0xc010914000?, 0x50c7cc0?, 0x10000000001?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0049f0bc8 sp=0xc0049f0b80 pc=0x7a20c9 net.(*conn).Read(0xc013f35a90, {0xc010914000?, 0xc00c91df00?, 0xc0106c0120?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0049f0c10 sp=0xc0049f0bc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc011c244e0, {0xc00390c900, 0x9, 0xc0056b268c?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc0049f0c48 sp=0xc0049f0c10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc011c244e0}, {0xc00390c900, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc0049f0c90 sp=0xc0049f0c48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc00390c900?, 0x9?, 0xc0106c0000?}, {0x62052c0?, 0xc011c244e0?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc0049f0ce0 sp=0xc0049f0c90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00390c8c0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc0049f0d90 sp=0xc0049f0ce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc011c20f00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc0049f0fc8 sp=0xc0049f0d90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc0049f0fe0 sp=0xc0049f0fc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0049f0fe8 sp=0xc0049f0fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 115743 [semacquire, 6 minutes]: runtime.gopark(0x18?, 0xc003440400?, 0x40?, 0xb4?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00398faa8 sp=0xc00398fa88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc00c7c52d0, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00398fb10 sp=0xc00398faa8 pc=0x4a17de sync.runtime_Semacquire(0xc00c7c52c0?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc00398fb40 sp=0xc00398fb10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc00398fb68 sp=0xc00398fb40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc00c7c52c0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc00398fb80 sp=0xc00398fb68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00c7c52c0?, {0x624a8a0?, 0xc00c7c5280?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc00398fbb0 sp=0xc00398fb80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc00c7c4800}, {0xc00c7c51c0, 0x1, 0xc0037887e0?}, 0x1, 0xc002e6b9e0, {0xc015333aa0, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc00398fd38 sp=0xc00398fbb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc000cd0ed0?, {0x624a8a0, 0xc00c7c4800}, {0xc010d81da0, 0x1, 0x1a00?}, {0xc00430d800?, 0x9c15540?, 0x0?}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc00398fe58 sp=0xc00398fd38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000cd0f30?, {0x624a8a0?, 0xc00c7c4800?}, {0xc010d81da0?, 0x1a01418?, 0x106663600?}, {0xc00b934d90?, 0xcd0f50?, 0xc0?}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc00398fed0 sp=0xc00398fe58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc00c7c4800?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc00398ff58 sp=0xc00398fed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00398ff78 sp=0xc00398ff58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00398ffe0 sp=0xc00398ff78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00398ffe8 sp=0xc00398ffe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 120618 [select, 6 minutes]: runtime.gopark(0xc00b34fb70?, 0x2?, 0x0?, 0x0?, 0xc00b34fb6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b34f9f8 sp=0xc00b34f9d8 pc=0x4907d6 runtime.selectgo(0xc00b34fb70, 0xc00b34fb68, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00b34fb38 sp=0xc00b34f9f8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc012017560) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00b34fba0 sp=0xc00b34fb38 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc015308210, {0x512dba0?, 0xc0111d7920}, 0xc007335440?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00b34fca8 sp=0xc00b34fba0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00b34fcd8 sp=0xc00b34fca8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc012017320, 0xc00b34fdb0, 0xc00b34fd80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc00b34fd30 sp=0xc00b34fcd8 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc012017320, {0x512dba0?, 0xc0111d7920?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00b34fde0 sp=0xc00b34fd30 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc016717b00, {0x512dba0?, 0xc0111d7920?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc00b34fe28 sp=0xc00b34fde0 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0161b1d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:132 +0x4c fp=0xc00b34fe58 sp=0xc00b34fe28 pc=0x1d4b00c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x628a900, 0xc0161b1d20}, 0xc000c62700, {0x624a948, 0xc0111d78c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:409 +0x5b fp=0xc00b34fee0 sp=0xc00b34fe58 pc=0x1d843fb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x624a948?, 0xc0111d78c0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:424 +0x45 fp=0xc00b34ff30 sp=0xc00b34fee0 pc=0x1d84345 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00b34ffe0 sp=0xc00b34ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b34ffe8 sp=0xc00b34ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498256 [select]: runtime.gopark(0xc00462fc18?, 0x3?, 0xd2?, 0xf9?, 0xc00462fb3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00462f8f0 sp=0xc00462f8d0 pc=0x4907d6 runtime.selectgo(0xc00462fc18, 0xc00462fb34, 0xc00df5ab00?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00462fa30 sp=0xc00462f8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc014d5de30}, {0x6221560, 0xc0057a6c60}, {0x46?, {0xc00462fe01?, 0x462fe60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00462fda0 sp=0xc00462fa30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc014d5de30?}, 0x164, 0xc0057a6c60, 0x1?, 0xc00096b410) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00462fe70 sp=0xc00462fda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc014d5de30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00462ff30 sp=0xc00462fe70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00462ffe0 sp=0xc00462ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00462ffe8 sp=0xc00462ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498492 [select]: runtime.gopark(0xc00ade9208?, 0x2?, 0xf0?, 0xa6?, 0xc00ade9204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00ade9090 sp=0xc00ade9070 pc=0x4907d6 runtime.selectgo(0xc00ade9208, 0xc00ade9200, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00ade91d0 sp=0xc00ade9090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc02d888a20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00ade9238 sp=0xc00ade91d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0308269a0, {0x51b9a20?, 0xc0533f6480}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00ade9340 sp=0xc00ade9238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x12a?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00ade9370 sp=0xc00ade9340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc02d8887e0, 0xc00ade9448, 0xc00ade9418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00ade93c8 sp=0xc00ade9370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc02d8887e0, {0x51b9a20?, 0xc0533f6480?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00ade9478 sp=0xc00ade93c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00564def0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0172273b0}, {0x51b9a20, 0xc0533f6480}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00ade94e0 sp=0xc00ade9478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00564def0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0172273b0}, {0x51b9a20, 0xc0533f6480}, 0x8f24b7f3a677e04?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00ade95d0 sp=0xc00ade94e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00564def0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0172273b0?}, {0x51b9a20?, 0xc0533f6480?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00ade9670 sp=0xc00ade95d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277410, {0x624a948, 0xc00564def0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00ade96f0 sp=0xc00ade9670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277410?}}, {0x624a948, 0xc00564def0}, 0xc0172273b0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00ade97c8 sp=0xc00ade96f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00564def0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc00ade9818 sp=0xc00ade97c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00fae8240, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00ade9ab8 sp=0xc00ade9818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00fae8240, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e497cc5c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00ade9bf0 sp=0xc00ade9ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e497cc5c, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00adea338 sp=0xc00ade9bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e497cc5c, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00adeac38 sp=0xc00adea338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e497cc5c, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00adeb4a0 sp=0xc00adeac38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e497cc5c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00adeb8c0 sp=0xc00adeb4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00adeb9f8 sp=0xc00adeb8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00564de90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00adebc00 sp=0xc00adeb9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00564de90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00adebf30 sp=0xc00adebc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00adebfe0 sp=0xc00adebf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00adebfe8 sp=0xc00adebfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498392 [select]: runtime.gopark(0xc0101fb808?, 0x3?, 0x3?, 0x0?, 0xc0101fb4d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0101fb258 sp=0xc0101fb238 pc=0x4907d6 runtime.selectgo(0xc0101fb808, 0xc0101fb4cc, 0x17600a769f018626?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0101fb398 sp=0xc0101fb258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc03b5dc5a0}, 0xc0134f03c0, 0xc004ef74a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc0101fbb08 sp=0xc0101fb398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc03b5dc5a0}, 0xc0134f03c0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc0101fbe28 sp=0xc0101fbb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a769d1a355d, 0x0, 0x0}, 0x0, {0x17600a76e030625f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc0101fc1f0 sp=0xc0101fbe28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a769d1a355d, 0x0, 0x0}, 0x0, {0x17600a76e030625f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0101fcb78 sp=0xc0101fc1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e030625f, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0101fcdc0 sp=0xc0101fcb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc03b5dc3c0}, {0x459c3d?}, 0xc0134f02d0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0101fd430 sp=0xc0101fcdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc03b5dc000}, 0xc0134f02d0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0101fd4d8 sp=0xc0101fd430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc03b5dc000}, {0x52f0080?, 0xc0134f02d0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc0101fd518 sp=0xc0101fd4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc03b5dc000}, {0x52f0080, 0xc0134f02d0}, 0xc02acfd320, 0xc01506c120) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0101fd848 sp=0xc0101fd518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03b5dc000?}, {0x52f0080?, 0xc0134f02d0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc0101fd888 sp=0xc0101fd848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc03b5dc000}, {0x52f0080, 0xc0134f02d0}, 0xc0101fd920?, 0xc00b400b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0101fd8b8 sp=0xc0101fd888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03b5dc000?}, {0x52f0080?, 0xc0134f02d0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0101fd8f8 sp=0xc0101fd8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc03b5dc000?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0101fd930 sp=0xc0101fd8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc03b5dc000}, {0x20300a?, 0x40?}, 0xc0101fd9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0101fd9a8 sp=0xc0101fd930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc03b5dc000?}, {0x52f0080?, 0xc0134f02d0?}, 0x5003b00?, 0x7f278cfaa501?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0101fda20 sp=0xc0101fd9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc03b5dc000?}, {0x52f0080?, 0xc0134f02d0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0101fda60 sp=0xc0101fda20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc03b5dc000}, {0x52f0080, 0xc0134f02d0}, 0xc02acfd320, 0xc01506c120) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc0101fdab0 sp=0xc0101fda60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc03b5dc000}, 0xc008428180, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc0101fdb08 sp=0xc0101fdab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc003ffd440, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc0101fde48 sp=0xc0101fdb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc003ffd440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc0101fdf68 sp=0xc0101fde48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc0101fdfe0 sp=0xc0101fdf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0101fdfe8 sp=0xc0101fdfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 423427 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x212?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f7a8f58 sp=0xc00f7a8f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcbdca0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00f7a8f90 sp=0xc00f7a8f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbdbb8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00f7a8fb0 sp=0xc00f7a8f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc031055b00?, 0xc007272000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00f7a8fd8 sp=0xc00f7a8fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc031055b00, {0xc007272000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00f7a9058 sp=0xc00f7a8fd8 pc=0x535efa net.(*netFD).Read(0xc031055b00, {0xc007272000?, 0x7b5a68?, 0xc031055b00?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00f7a90a0 sp=0xc00f7a9058 pc=0x7a20c9 net.(*conn).Read(0xc000549708, {0xc007272000?, 0x164290f?, 0xc000549708?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00f7a90e8 sp=0xc00f7a90a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00dfc04e0, {0xc007272000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00f7a9130 sp=0xc00f7a90e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03cf79b00, {0xc007272000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00f7a91d0 sp=0xc00f7a9130 pc=0x370f896 bufio.(*Reader).fill(0xc00d8c2090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00f7a9208 sp=0xc00f7a91d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00d8c2090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00f7a9228 sp=0xc00f7a9208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00d8c2668?, {0x62295c8, 0xc00d8c2090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00f7a9260 sp=0xc00f7a9228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00d8c2000, 0xc011ce8980, 0xc00f7a9442, 0xc00f7a9444, {0x623b718, 0xc00dd2ce10}, 0xc0059fa000, 0xc00d9ea1d4, 0xc00f7a9440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00f7a9398 sp=0xc00f7a9260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00d8c2000, {0x624a948, 0xc013754ff0}, 0xc011ce8970, 0xc0059fa000, 0xc013754e70, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00f7a9768 sp=0xc00f7a9398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc013754ff0}, {0x628a538, 0xc00dfc04e0}, {{{0xc006d96022, 0x4}}, 0x1, 0xc013754f60, 0xc013754f90, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00f7a98a8 sp=0xc00f7a9768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc013754ff0}, {0x628a538?, 0xc00dfc04e0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00f7a9e20 sp=0xc00f7a98a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc013754d50?}, {0x628a538, 0xc00dfc04e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00f7a9eb0 sp=0xc00f7a9e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc013754d50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00f7a9f30 sp=0xc00f7a9eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f7a9fe0 sp=0xc00f7a9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f7a9fe8 sp=0xc00f7a9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498383 [select]: runtime.gopark(0xc00a355c18?, 0x3?, 0xd2?, 0x59?, 0xc00a355b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a3558f0 sp=0xc00a3558d0 pc=0x4907d6 runtime.selectgo(0xc00a355c18, 0xc00a355b34, 0xc011cb10c0?, 0x0, 0x7?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a355a30 sp=0xc00a3558f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc01847a360}, {0x6221560, 0xc004b417a0}, {0x46?, {0xc00a355e01?, 0xa355e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00a355da0 sp=0xc00a355a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc01847a360?}, 0x4f8, 0xc004b417a0, 0x0?, 0xc018974c70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00a355e70 sp=0xc00a355da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc01847a360}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00a355f30 sp=0xc00a355e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a355fe0 sp=0xc00a355f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a355fe8 sp=0xc00a355fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 492433 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x28d?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011fd8f58 sp=0xc011fd8f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f300df8?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc011fd8f90 sp=0xc011fd8f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300d10, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc011fd8fb0 sp=0xc011fd8f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00d503100?, 0xc004cf6000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc011fd8fd8 sp=0xc011fd8fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00d503100, {0xc004cf6000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc011fd9058 sp=0xc011fd8fd8 pc=0x535efa net.(*netFD).Read(0xc00d503100, {0xc004cf6000?, 0x7b5a68?, 0xc00d503100?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc011fd90a0 sp=0xc011fd9058 pc=0x7a20c9 net.(*conn).Read(0xc0040812d8, {0xc004cf6000?, 0x164290f?, 0xc0040812d8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc011fd90e8 sp=0xc011fd90a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00ad68600, {0xc004cf6000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc011fd9130 sp=0xc011fd90e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc013d82ea0, {0xc004cf6000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc011fd91d0 sp=0xc011fd9130 pc=0x370f896 bufio.(*Reader).fill(0xc002a17090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc011fd9208 sp=0xc011fd91d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc002a17090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc011fd9228 sp=0xc011fd9208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc002a17668?, {0x62295c8, 0xc002a17090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc011fd9260 sp=0xc011fd9228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc002a17000, 0xc011ce81f0, 0xc011fd9442, 0xc011fd9444, {0x623b718, 0xc0078d4990}, 0xc0059fa000, 0xc011d9b754, 0xc011fd9440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc011fd9398 sp=0xc011fd9260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc002a17000, {0x624a948, 0xc003d31260}, 0xc011ce81d0, 0xc0059fa000, 0xc003d310b0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc011fd9768 sp=0xc011fd9398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc003d31260}, {0x628a538, 0xc00ad68600}, {{{0xc004a16038, 0x4}}, 0x1, 0xc003d310e0, 0xc003d31140, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc011fd98a8 sp=0xc011fd9768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc003d31260}, {0x628a538?, 0xc00ad68600}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc011fd9e20 sp=0xc011fd98a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc003d30f00?}, {0x628a538, 0xc00ad68600}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc011fd9eb0 sp=0xc011fd9e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc003d30f00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc011fd9f30 sp=0xc011fd9eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc011fd9fe0 sp=0xc011fd9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011fd9fe8 sp=0xc011fd9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 407459 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x4d4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012066f58 sp=0xc012066f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb6fa0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc012066f90 sp=0xc012066f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb6eb8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc012066fb0 sp=0xc012066f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc0124da600?, 0xc00f61b000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc012066fd8 sp=0xc012066fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0124da600, {0xc00f61b000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc012067058 sp=0xc012066fd8 pc=0x535efa net.(*netFD).Read(0xc0124da600, {0xc00f61b000?, 0x7b5a68?, 0xc0124da600?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0120670a0 sp=0xc012067058 pc=0x7a20c9 net.(*conn).Read(0xc00d97e0b0, {0xc00f61b000?, 0x164290f?, 0xc00d97e0b0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0120670e8 sp=0xc0120670a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00799c2a0, {0xc00f61b000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc012067130 sp=0xc0120670e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03cf79a40, {0xc00f61b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc0120671d0 sp=0xc012067130 pc=0x370f896 bufio.(*Reader).fill(0xc00d3b3490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc012067208 sp=0xc0120671d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00d3b3490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc012067228 sp=0xc012067208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00d3b3a68?, {0x62295c8, 0xc00d3b3490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc012067260 sp=0xc012067228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00d3b3400, 0xc00be504b0, 0xc012067442, 0xc012067444, {0x623b718, 0xc0124261b0}, 0xc0059fa000, 0xc00c83dd94, 0xc012067440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc012067398 sp=0xc012067260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00d3b3400, {0x624a948, 0xc046736f60}, 0xc00be504a0, 0xc0059fa000, 0xc0467366f0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc012067768 sp=0xc012067398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc046736f60}, {0x628a538, 0xc00799c2a0}, {{{0xc00f34d038, 0x4}}, 0x1, 0xc046736720, 0xc046736750, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc0120678a8 sp=0xc012067768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc046736f60}, {0x628a538?, 0xc00799c2a0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc012067e20 sp=0xc0120678a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc046736630?}, {0x628a538, 0xc00799c2a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc012067eb0 sp=0xc012067e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc046736630}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc012067f30 sp=0xc012067eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012067fe0 sp=0xc012067f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012067fe8 sp=0xc012067fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498319 [select]: runtime.gopark(0xc004203c18?, 0x3?, 0xd2?, 0x39?, 0xc004203b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0042038f0 sp=0xc0042038d0 pc=0x4907d6 runtime.selectgo(0xc004203c18, 0xc004203b34, 0xc0081523c0?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004203a30 sp=0xc0042038f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc0034e9950}, {0x6221560, 0xc00dd63c20}, {0x46?, {0xc004203e01?, 0x4203e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc004203da0 sp=0xc004203a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc0034e9950?}, 0x164, 0xc00dd63c20, 0xa0?, 0xc0120e6b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc004203e70 sp=0xc004203da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc0034e9950}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc004203f30 sp=0xc004203e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc004203fe0 sp=0xc004203f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004203fe8 sp=0xc004203fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498332 [select]: runtime.gopark(0xc00f165208?, 0x2?, 0x18?, 0xb8?, 0xc00f165204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f165090 sp=0xc00f165070 pc=0x4907d6 runtime.selectgo(0xc00f165208, 0xc00f165200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f1651d0 sp=0xc00f165090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc012405e60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00f165238 sp=0xc00f1651d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc03bf94d10, {0x51b9a20?, 0xc0130d3a80}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00f165340 sp=0xc00f165238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00f165370 sp=0xc00f165340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0124059e0, 0xc00f165448, 0xc00f165418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00f1653c8 sp=0xc00f165370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0124059e0, {0x51b9a20?, 0xc0130d3a80?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00f165478 sp=0xc00f1653c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00fb7a360?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0048522d0}, {0x51b9a20, 0xc0130d3a80}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00f1654e0 sp=0xc00f165478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00fb7a360}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0048522d0}, {0x51b9a20, 0xc0130d3a80}, 0x54ea01ef14316b1a?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00f1655d0 sp=0xc00f1654e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00fb7a360?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0048522d0?}, {0x51b9a20?, 0xc0130d3a80?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00f165670 sp=0xc00f1655d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043276528, {0x624a948, 0xc00fb7a360}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00f1656f0 sp=0xc00f165670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043276528?}}, {0x624a948, 0xc00fb7a360}, 0xc0048522d0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00f1657c8 sp=0xc00f1656f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00fb7a360?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc00f165818 sp=0xc00f1657c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0040e2660, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f165ab8 sp=0xc00f165818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0040e2660, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76dea157ce, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f165bf0 sp=0xc00f165ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76dea157ce, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f166338 sp=0xc00f165bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76dea157ce, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f166c38 sp=0xc00f166338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76dea157ce, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f1674a0 sp=0xc00f166c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76dea157ce, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f1678c0 sp=0xc00f1674a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00f1679f8 sp=0xc00f1678c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00fb7a330}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00f167c00 sp=0xc00f1679f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00fb7a330}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00f167f30 sp=0xc00f167c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f167fe0 sp=0xc00f167f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f167fe8 sp=0xc00f167fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 418770 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x507?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010004f58 sp=0xc010004f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539810?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc010004f90 sp=0xc010004f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539728, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc010004fb0 sp=0xc010004f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc01956f100?, 0xc00d46e000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc010004fd8 sp=0xc010004fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01956f100, {0xc00d46e000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc010005058 sp=0xc010004fd8 pc=0x535efa net.(*netFD).Read(0xc01956f100, {0xc00d46e000?, 0x7b5a68?, 0xc01956f100?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0100050a0 sp=0xc010005058 pc=0x7a20c9 net.(*conn).Read(0xc0061edd50, {0xc00d46e000?, 0x164290f?, 0xc0061edd50?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0100050e8 sp=0xc0100050a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc013898000, {0xc00d46e000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc010005130 sp=0xc0100050e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc011624b70, {0xc00d46e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc0100051d0 sp=0xc010005130 pc=0x370f896 bufio.(*Reader).fill(0xc003b95c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc010005208 sp=0xc0100051d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003b95c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc010005228 sp=0xc010005208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003b96268?, {0x62295c8, 0xc003b95c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc010005260 sp=0xc010005228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003b95c00, 0xc00faa3a90, 0xc010005442, 0xc010005444, {0x623b718, 0xc011db8d80}, 0xc0059fa000, 0xc014dfa504, 0xc010005440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc010005398 sp=0xc010005260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003b95c00, {0x624a948, 0xc01578e240}, 0xc00faa3a80, 0xc0059fa000, 0xc01578e030, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc010005768 sp=0xc010005398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc01578e240}, {0x628a538, 0xc013898000}, {{{0xc00a4aa046, 0x4}}, 0x1, 0xc01578e090, 0xc01578e0c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc0100058a8 sp=0xc010005768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc01578e240}, {0x628a538?, 0xc013898000}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc010005e20 sp=0xc0100058a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc00de55e00?}, {0x628a538, 0xc013898000}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc010005eb0 sp=0xc010005e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc00de55e00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc010005f30 sp=0xc010005eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc010005fe0 sp=0xc010005f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010005fe8 sp=0xc010005fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 120412 [select, 6 minutes]: runtime.gopark(0xc007e25fb0?, 0x2?, 0x0?, 0x6e?, 0xc007e25f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0049eae18 sp=0xc0049eadf8 pc=0x4907d6 runtime.selectgo(0xc0049eafb0, 0xc007e25f90, 0xc00296b6c0?, 0x0, 0xc0038e3380?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0049eaf58 sp=0xc0049eae18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0049eafe0 sp=0xc0049eaf58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0049eafe8 sp=0xc0049eafe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 423428 [select]: runtime.gopark(0xc016b66128?, 0x2?, 0xd8?, 0xb2?, 0xc016b66124?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016b65fb0 sp=0xc016b65f90 pc=0x4907d6 runtime.selectgo(0xc016b66128, 0xc016b66120, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc016b660f0 sp=0xc016b65fb0 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc015391e60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc016b66158 sp=0xc016b660f0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0037b2580, {0x51b9a20?, 0xc011420980}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc016b66260 sp=0xc016b66158 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x1a8?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc016b66290 sp=0xc016b66260 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc015391c20, 0xc016b66368, 0xc016b66338) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc016b662e8 sp=0xc016b66290 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc015391c20, {0x51b9a20?, 0xc011420980?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc016b66398 sp=0xc016b662e8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc040030390?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0106174a0}, {0x51b9a20, 0xc011420980}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc016b66400 sp=0xc016b66398 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc040030390}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0106174a0}, {0x51b9a20, 0xc011420980}, 0x368db5891efddb91?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc016b664f0 sp=0xc016b66400 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x432e6dbacb843a6b?, {0x624a948?, 0xc040030390?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0106174a0?}, {0x51b9a20?, 0xc011420980?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc016b66590 sp=0xc016b664f0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc046762478, {0x624a948, 0xc040030390}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc016b66610 sp=0xc016b66590 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc046762478?}}, {0x624a948, 0xc040030390}, 0xc0106174a0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc016b666e8 sp=0xc016b66610 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x16b66750?, {0x624a948?, 0xc040030390?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc016b66848?}) :1 +0x65 fp=0xc016b66738 sp=0xc016b666e8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc03d8596e0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc016b669d8 sp=0xc016b66738 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc03d8596e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5045f6e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc016b66b10 sp=0xc016b669d8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5045f6e, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc016b67258 sp=0xc016b66b10 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5045f6e, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc016b67b58 sp=0xc016b67258 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5045f6e, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc016b683c0 sp=0xc016b67b58 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e5045f6e, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc016b687e0 sp=0xc016b683c0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0146fd6b8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc016b68a38 sp=0xc016b687e0 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0146fd680, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc016b68b60 sp=0xc016b68a38 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0146fd650, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc016b68df0 sp=0xc016b68b60 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0146fd550, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc016b69390 sp=0xc016b68df0 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0146fd550, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc016b69680 sp=0xc016b69390 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0146fd420, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc016b69980 sp=0xc016b69680 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0146fd400?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc016b69b28 sp=0xc016b69980 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0146fd350, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc016b69d40 sp=0xc016b69b28 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0146fd180, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc016b69f98 sp=0xc016b69d40 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc016b6a0c0 sp=0xc016b69f98 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc005ba51e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc016b6a388 sp=0xc016b6a0c0 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:233 +0x85 fp=0xc016b6a4a0 sp=0xc016b6a388 pc=0x2332605 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00ad15080, {0x624a948, 0xc040030180}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:468 +0x346 fp=0xc016b6a6c0 sp=0xc016b6a4a0 pc=0x2333346 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00ad15080, {0x624a948, 0xc040030180}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:671 +0x1145 fp=0xc016b6a9b8 sp=0xc016b6a6c0 pc=0x23347e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc011f06a80, {0x624a948, 0xc040030180}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:264 +0xde fp=0xc016b6aa98 sp=0xc016b6a9b8 pc=0x233723e github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).nextKey(0xc004801b80, {0x624a948?, 0xc040030180?}) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:665 +0x7b fp=0xc016b6ac48 sp=0xc016b6aa98 pc=0x232b69b github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).startScan(...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:635 github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).StartScan(0xc004801b80, {0x624a948, 0xc040030180}, {0xc00ad15200, 0x1?, 0x203003?}, {0xc002626140?, 0x11?, 0x45e42d?}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:492 +0x18f fp=0xc016b6acc0 sp=0xc016b6ac48 pc=0x232a82f github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc0014daa00) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:999 +0xbca fp=0xc016b6aea0 sp=0xc016b6acc0 pc=0x258acca github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc0014daa00) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:731 +0x70 fp=0xc016b6af20 sp=0xc016b6aea0 pc=0x25899d0 github.com/cockroachdb/cockroach/pkg/sql.(*rowSourceToPlanNode).Next(0xc00b90db00, {{0x0?, 0x0?}, 0xc0400301b0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/sql/row_source_to_plan_node.go:79 +0x45 fp=0xc016b6af98 sp=0xc016b6af20 pc=0x34b4f05 github.com/cockroachdb/cockroach/pkg/sql.(*errorIfRowsNode).Next(0xc013f60d80, {{0x624a948?, 0xc040030120?}, 0xc006543800?, 0xc00404c578?}) github.com/cockroachdb/cockroach/pkg/sql/error_if_rows.go:42 +0x4b fp=0xc016b6afd0 sp=0xc016b6af98 pc=0x33e49cb github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc011e35b80) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:211 +0x1b9 fp=0xc016b6b0c8 sp=0xc016b6afd0 pc=0x3479e19 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc0027d8000) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:243 +0x13c fp=0xc016b6b448 sp=0xc016b6b0c8 pc=0x299d6fc github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(0xc006dc2a50) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:252 +0x2a fp=0xc016b6b460 sp=0xc016b6b448 pc=0x32257ca github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter-fm() :1 +0x26 fp=0xc016b6b478 sp=0xc016b6b460 pc=0x32334e6 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc000ef2380?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc016b6b4b8 sp=0xc016b6b478 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:256 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc006dc2a50, {0x624a8a0?, 0xc003b9d7c0?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:299 +0x308 fp=0xc016b6b648 sp=0xc016b6b4b8 pc=0x3225c08 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc01266e9b0, {0x624a8a0, 0xc003b9d7c0}, 0xc005ba51e0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:320 +0x1db fp=0xc016b6b6e8 sp=0xc016b6b648 pc=0x322ac3b github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc0222c99b0}, 0xc010616ff0, 0xc005ba51e0, 0xc011420580, 0xc008e5ee00, 0xc01537b000, 0xc011d94a20) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc016b6bf98 sp=0xc016b6b6e8 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).planAndRunPostquery(0xc0052d77c0, {0x624a948, 0xc0222c99b0}, {{0x624c6d0?, 0xc013f60d80?}, 0x0?}, 0xc00404c578, 0xc01537b000, 0x33b057f?, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:2044 +0x71e fp=0xc016b6c148 sp=0xc016b6bf98 pc=0x33bfdfe github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunCascadesAndChecks(0xc0052d77c0, {0x624a948, 0xc0222c99b0}, 0xc00404c578, 0xc011d4a940, 0xc00404ccd8, 0xc0077fe000) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1931 +0xe8c fp=0xc016b6c2f8 sp=0xc016b6c148 pc=0x33bf44c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc00404c000?, {0x624a948, 0xc0222c99b0}, 0xc00404c8b0, 0xc00404c578?, 0xc00404c578, 0xc0077fe000, 0xc011d4a940) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1488 +0x3aa fp=0xc016b6c410 sp=0xc016b6c2f8 pc=0x33bc68a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00404c000, {0x624a948?, 0xc0222c99b0}, 0xc00404c578, 0xc0222c99b0?, {0x7f275f329680?, 0xc00d8c26c8}, 0xa7?, 0xc00d8f0ec8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc016b6c518 sp=0xc016b6c410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00404c000, {0x624a8a0, 0xc008153d00}, 0xc00404c578, {0x7f275f329680, 0xc00d8c26c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc016b6c938 sp=0xc016b6c518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00404c000, {0x624a948?, 0xc042b6cff0?}, {{0x6273140, 0xc006883180}, {0xc0077b2495, 0x283}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc016b6d2c8 sp=0xc016b6c938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc042b6cff0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc016b6d380 sp=0xc016b6d2c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc042b6cff0?}, {0x6273140?, 0xc006883180?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc016b6d4d0 sp=0xc016b6d380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00404c000, {0x624a948, 0xc042b6cff0}, {{0x6273140, 0xc006883180}, {0xc0077b2495, 0x283}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc016b6d648 sp=0xc016b6d4d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6273140, 0xc006883180}, {0xc0077b2495, 0x283}, 0x0, 0x1}, {0xc1116e81932b4bba, 0x7fe91a7a10, 0x0}, {0xc1116e81932b5512, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc016b6d708 sp=0xc016b6d648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00404c000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc016b6dcb8 sp=0xc016b6d708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00404c000, {0x624a8a0, 0xc0035197c0}, 0x411638a400000000?, 0xc004e8fdc0?, 0xc011ce89b0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc016b6dd70 sp=0xc016b6dcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc0035197c0?}, {0xc0059fa000?}, 0x3?, 0xc003f3d728?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc016b6de20 sp=0xc016b6dd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc016b6dfe0 sp=0xc016b6de20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016b6dfe8 sp=0xc016b6dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 427179 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x444?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012d3ef58 sp=0xc012d3ef38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64539720?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc012d3ef90 sp=0xc012d3ef58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764539638, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc012d3efb0 sp=0xc012d3ef90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc04742ef00?, 0xc007a7a000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc012d3efd8 sp=0xc012d3efb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04742ef00, {0xc007a7a000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc012d3f058 sp=0xc012d3efd8 pc=0x535efa net.(*netFD).Read(0xc04742ef00, {0xc007a7a000?, 0x7b5a68?, 0xc04742ef00?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc012d3f0a0 sp=0xc012d3f058 pc=0x7a20c9 net.(*conn).Read(0xc046762df8, {0xc007a7a000?, 0x164290f?, 0xc046762df8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc012d3f0e8 sp=0xc012d3f0a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00559fe60, {0xc007a7a000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc012d3f130 sp=0xc012d3f0e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0582fc3a8, {0xc007a7a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc012d3f1d0 sp=0xc012d3f130 pc=0x370f896 bufio.(*Reader).fill(0xc005ca9890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc012d3f208 sp=0xc012d3f1d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc005ca9890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc012d3f228 sp=0xc012d3f208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc005ca9e68?, {0x62295c8, 0xc005ca9890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc012d3f260 sp=0xc012d3f228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc005ca9800, 0xc00d467990, 0xc012d3f442, 0xc012d3f444, {0x623b718, 0xc00141a360}, 0xc0059fa000, 0xc015b39c24, 0xc012d3f440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc012d3f398 sp=0xc012d3f260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc005ca9800, {0x624a948, 0xc01effe600}, 0xc00d467980, 0xc0059fa000, 0xc01effe540, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc012d3f768 sp=0xc012d3f398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc01effe600}, {0x628a538, 0xc00559fe60}, {{{0xc007a6e038, 0x4}}, 0x1, 0xc01effe570, 0xc01effe5a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc012d3f8a8 sp=0xc012d3f768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc01effe600}, {0x628a538?, 0xc00559fe60}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc012d3fe20 sp=0xc012d3f8a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc01effe480?}, {0x628a538, 0xc00559fe60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc012d3feb0 sp=0xc012d3fe20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc01effe480}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc012d3ff30 sp=0xc012d3feb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012d3ffe0 sp=0xc012d3ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012d3ffe8 sp=0xc012d3ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115744 [select]: runtime.gopark(0xc00187fe18?, 0x2?, 0x28?, 0x4a?, 0xc00187fdfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00187fc68 sp=0xc00187fc48 pc=0x4907d6 runtime.selectgo(0xc00187fe18, 0xc00187fdf8, 0xc00a683e80?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00187fda8 sp=0xc00187fc68 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc006663600, {0x624a8a0, 0xc00c7c4800}, 0xc010d81e00?, 0xc002e6b9e0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:351 +0xdf fp=0xc00187ff20 sp=0xc00187fda8 pc=0x1a0167f github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x624a8a0?, 0xc00c7c4800?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:308 +0x38 fp=0xc00187ff58 sp=0xc00187ff20 pc=0x1a01418 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00187ff78 sp=0xc00187ff58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00187ffe0 sp=0xc00187ff78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00187ffe8 sp=0xc00187ffe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 423288 [select]: runtime.gopark(0xc016d6ce00?, 0x2?, 0x80?, 0x2a?, 0xc016d6cdfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016d6cc88 sp=0xc016d6cc68 pc=0x4907d6 runtime.selectgo(0xc016d6ce00, 0xc016d6cdf8, 0x10?, 0x0, 0x301?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc016d6cdc8 sp=0xc016d6cc88 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc01104a6c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc016d6ce30 sp=0xc016d6cdc8 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018add90, {0x51b9a20?, 0xc011e78580}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc016d6cf38 sp=0xc016d6ce30 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x338?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc016d6cf68 sp=0xc016d6cf38 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc01104a360, 0xc016d6d040, 0xc016d6d010) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc016d6cfc0 sp=0xc016d6cf68 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc01104a360, {0x51b9a20?, 0xc011e78580?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc016d6d070 sp=0xc016d6cfc0 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc019ee4f60?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc001c480f0}, {0x51b9a20, 0xc011e78580}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc016d6d0d8 sp=0xc016d6d070 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc019ee4f60}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc001c480f0}, {0x51b9a20, 0xc011e78580}, 0x1a0298d41431620f?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc016d6d1c8 sp=0xc016d6d0d8 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x7a808f5bc3caccbe?, {0x624a948?, 0xc019ee4f60?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc001c480f0?}, {0x51b9a20?, 0xc011e78580?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc016d6d268 sp=0xc016d6d1c8 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc012277510, {0x624a948, 0xc019ee4f60}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc016d6d2e8 sp=0xc016d6d268 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc012277510?}}, {0x624a948, 0xc019ee4f60}, 0xc001c480f0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc016d6d3c0 sp=0xc016d6d2e8 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0x16d6d428?, {0x624a948?, 0xc019ee4f60?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc016d6d520?}) :1 +0x65 fp=0xc016d6d410 sp=0xc016d6d3c0 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00dc49f20, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc016d6d6b0 sp=0xc016d6d410 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00dc49f20, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f9b879, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc016d6d7e8 sp=0xc016d6d6b0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f9b879, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc016d6df30 sp=0xc016d6d7e8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f9b879, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc016d6e830 sp=0xc016d6df30 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f9b879, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc016d6f098 sp=0xc016d6e830 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3f9b879, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc016d6f4b8 sp=0xc016d6f098 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc007b5dc38, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc016d6f710 sp=0xc016d6f4b8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc007b5dc00, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc016d6f838 sp=0xc016d6f710 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc007b5dbd0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x746 fp=0xc016d6fac8 sp=0xc016d6f838 pc=0x19e5026 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc007b5dad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:225 +0x27a fp=0xc016d70068 sp=0xc016d6fac8 pc=0x19eeefa github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc007b5dad0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:153 +0x1cb fp=0xc016d70358 sp=0xc016d70068 pc=0x19ee7ab github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc007b5d9a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x2ee fp=0xc016d70658 sp=0xc016d70358 pc=0x19e986e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc007b5d980?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:104 +0xb5 fp=0xc016d70800 sp=0xc016d70658 pc=0x19ee115 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc007b5d8d0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:245 +0x4e8 fp=0xc016d70a18 sp=0xc016d70800 pc=0x19e6c28 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc007b5d700, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:526 +0x592 fp=0xc016d70c70 sp=0xc016d70a18 pc=0x19dd5f2 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc016d70d98 sp=0xc016d70c70 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0134eedc0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1093 +0x225 fp=0xc016d71060 sp=0xc016d70d98 pc=0x16b7905 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send-fm({0x624a948?, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc016d71170 sp=0xc016d71060 pc=0x16bc4c5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc019ee4c30}, 0xc016d713d8?, 0xc0129e2680) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc016d71368 sp=0xc016d71170 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0134eedc0, {0x624a948, 0xc019ee4c30}, 0xc00a0f9400?) github.com/cockroachdb/cockroach/pkg/kv/txn.go:673 +0x74 fp=0xc016d713b0 sp=0xc016d71368 pc=0x16b5274 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc010b3cb28, {0x624a948, 0xc019ee4c30}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:230 +0xb6 fp=0xc016d713e8 sp=0xc016d713b0 pc=0x350dd96 github.com/cockroachdb/cockroach/pkg/sql.(*insertNode).BatchedNext(0xc010b3cb00, {{0x624a948, 0xc019ee4c30}, 0xc00a0f9400, 0xc0037ed178}) github.com/cockroachdb/cockroach/pkg/sql/insert.go:260 +0x22e fp=0xc016d71448 sp=0xc016d713e8 pc=0x341c12e github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00c0720d8, {{0x624a948?, 0xc019ee4c30?}, 0xc00a0f9400?, 0xc0037ed178?}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce fp=0xc016d714a0 sp=0xc016d71448 pc=0x34772ae github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc016d716a0?, 0x624c938?}, {0x624ceb0, 0xc00c0720d8}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:518 +0x143 fp=0xc016d71550 sp=0xc016d714a0 pc=0x3476843 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e fp=0xc016d71588 sp=0xc016d71550 pc=0x354d95e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc016d716a0, {0x624ceb0?, 0xc00c0720d8?}, {0x5338c31?, 0x5?}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c3 fp=0xc016d71618 sp=0xc016d71588 pc=0x354d863 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc016d716a0, {0x624ceb0, 0xc00c0720d8}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 fp=0xc016d71668 sp=0xc016d71618 pc=0x354c297 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x624a948?, 0xc019ee4c30?}, 0xc00a0f9400?, 0xc0037ed178?}, {0x624ceb0, 0xc00c0720d8}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:521 +0x119 fp=0xc016d716e8 sp=0xc016d71668 pc=0x3476699 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc011bfe000, {0x624a948, 0xc019ee4c00}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:166 +0xbd fp=0xc016d71728 sp=0xc016d716e8 pc=0x3479bfd github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:122 +0x34 fp=0xc016d71750 sp=0xc016d71728 pc=0x3224ef4 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc030825440?) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 fp=0xc016d71790 sp=0xc016d71750 pc=0x2126aa2 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc030825440, {0x624a8a0?, 0xc00d51f340?}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:121 +0x70 fp=0xc016d717e8 sp=0xc016d71790 pc=0x3224e70 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc030825440, {0x624a8a0?, 0xc00d51f340?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:728 +0x3a fp=0xc016d71828 sp=0xc016d717e8 pc=0x230ec1a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00dc0c200, {0x624a8a0, 0xc00d51f340}, 0xc030825440?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:480 +0x25a fp=0xc016d718e8 sp=0xc016d71828 pc=0x25bf07a github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x629ebe0?, {0x624a8a0?, 0xc00d51f340?}, 0xc0134eedc0?) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:307 +0x206 fp=0xc016d71988 sp=0xc016d718e8 pc=0x322ac66 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0052d77c0, {0x624a948, 0xc012668ba0}, 0xc0120d3950, 0xc0134eedc0, 0xc011e78080, 0xc00b0ef880, 0xc0037ed4b0, 0xc0134da2d0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:757 +0xbc4 fp=0xc016d72238 sp=0xc016d71988 pc=0x33b9844 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x6208b80?, {0x624a948?, 0xc012668ba0}, 0x203003?, 0xc0120d3950, 0x0?, {{0x624ceb0?, 0xc00c0720d8?}, 0x0?}, 0xc00b0ef880, ...) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1764 +0x265 fp=0xc016d722f8 sp=0xc016d72238 pc=0x33be445 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc0037ecc00?, {0x624a948, 0xc012668ba0}, 0xc0037ed4b0, 0xc0037ed178?, 0xc0037ed178, 0xc00b0ef880, 0xc00d51f280) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1472 +0x218 fp=0xc016d72410 sp=0xc016d722f8 pc=0x33bc4f8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0037ecc00, {0x624a948?, 0xc012668ba0}, 0xc0037ed178, 0xc012668ba0?, {0x7f275f329680?, 0xc00d3ae6c8}, 0xa9?, 0xc014d328a8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1597 +0x594 fp=0xc016d72518 sp=0xc016d72410 pc=0x3310cf4 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0037ecc00, {0x624a8a0, 0xc00d51e980}, 0xc0037ed178, {0x7f275f329680, 0xc00d3ae6c8}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1200 +0xc2f fp=0xc016d72938 sp=0xc016d72518 pc=0x330e08f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0037ecc00, {0x624a948?, 0xc0118bc4b0?}, {{0x6273140, 0xc011f85ef0}, {0xc00181c006, 0x407}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:694 +0x2374 fp=0xc016d732c8 sp=0xc016d72938 pc=0x3309b54 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x624a948?, 0xc0118bc4b0?}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131 +0xfc fp=0xc016d73380 sp=0xc016d732c8 pc=0x330725c github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x624a948?, {0x624a948?, 0xc0118bc4b0?}, {0x6273140?, 0xc011f85ef0?}, 0x0?, 0x0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2438 +0x330 fp=0xc016d734d0 sp=0xc016d73380 pc=0x3317010 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0037ecc00, {0x624a948, 0xc0118bc4b0}, {{0x6273140, 0xc011f85ef0}, {0xc00181c006, 0x407}, 0x0, 0x1}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0x4bf fp=0xc016d73648 sp=0xc016d734d0 pc=0x3306bff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x6273140, 0xc011f85ef0}, {0xc00181c006, 0x407}, 0x0, 0x1}, {0xc1116e8196c4e5ed, 0x7fecb4143e, 0x0}, {0xc1116e8196c4ee9b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2123 +0x306 fp=0xc016d73708 sp=0xc016d73648 pc=0x32fb626 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0037ecc00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2127 +0x1168 fp=0xc016d73cb8 sp=0xc016d73708 pc=0x32f9be8 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0037ecc00, {0x624a8a0, 0xc00d85ee80}, 0x4119859400000000?, 0xc001159dc0?, 0xc00ae1d0f0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc016d73d70 sp=0xc016d73cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00d85ee80?}, {0xc0059fa000?}, 0x3?, 0xc0038a6f30?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc016d73e20 sp=0xc016d73d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc016d73fe0 sp=0xc016d73e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016d73fe8 sp=0xc016d73fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 488386 [chan receive]: runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x203002?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001c71cd0 sp=0xc001c71cb0 pc=0x4907d6 runtime.chanrecv(0xc0044a06c0, 0xc0064f3e50, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc001c71d60 sp=0xc001c71cd0 pc=0x45a91b runtime.chanrecv2(0x40?, 0xc002100400?) GOROOT/src/runtime/chan.go:447 +0x18 fp=0xc001c71d88 sp=0xc001c71d60 pc=0x45a458 github.com/cockroachdb/cockroach/pkg/ccl/backupccl.(*backupDataProcessor).Next(0xc0172d6a80) github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backup_processor.go:194 +0x85 fp=0xc001c71f08 sp=0xc001c71d88 pc=0x3a84ac5 github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run({0x624a948, 0xc00e2eda70}, {0x626a6c0, 0xc0172d6a80}, {0x6222af0?, 0xc0085a5c00?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:186 +0x55 fp=0xc001c71f58 sp=0xc001c71f08 pc=0x2309bf5 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc0172d6a80, {0x624a948?, 0xc00e2ed0e0?}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:729 +0x5b fp=0xc001c71f98 sp=0xc001c71f58 pc=0x230ec3b github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).StartInternal.func1(0xc00bbcbf70?) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:432 +0x48 fp=0xc001c71fc8 sp=0xc001c71f98 pc=0x25bece8 github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).StartInternal.func2() github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:434 +0x2a fp=0xc001c71fe0 sp=0xc001c71fc8 pc=0x25bec6a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001c71fe8 sp=0xc001c71fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).StartInternal github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:431 +0x3d5 goroutine 422955 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc015e45640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc015e455d0 sp=0xc015e455b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00c594250, 0x1e0) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc015e45618 sp=0xc015e455d0 pc=0x4be30c sync.(*Cond).Wait(0xc00f35bc00?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc015e45650 sp=0xc015e45618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00f35c268) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc015e45708 sp=0xc015e45650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003bbc600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc015e45cb8 sp=0xc015e45708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003bbc600, {0x624a8a0, 0xc00c594280}, 0x4117000400000000?, 0xc004e8fdc0?, 0xc012353fa0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc015e45d70 sp=0xc015e45cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00c594280?}, {0xc0059fa000?}, 0x3?, 0xc0030bc6c0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc015e45e20 sp=0xc015e45d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc015e45fe0 sp=0xc015e45e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc015e45fe8 sp=0xc015e45fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 375091 [sleep]: runtime.gopark(0x4b8d30536a9?, 0x2050ab?, 0xba?, 0x5?, 0x80ca2c5c0c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003a6ddf0 sp=0xc003a6ddd0 pc=0x4907d6 time.Sleep(0x153399001) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003a6de30 sp=0xc003a6ddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc002eb4e70, 0x2050ab, {0x4618e7?, 0x203000?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003a6de88 sp=0xc003a6de30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc002eb4e70, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003a6dec8 sp=0xc003a6de88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00efcd500?, 0x2f, 0xc005a4df70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003a6dfa8 sp=0xc003a6dec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003a6dfe0 sp=0xc003a6dfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003a6dfe8 sp=0xc003a6dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 49081 [select, 7 minutes]: runtime.gopark(0xc003a17b70?, 0x2?, 0x47?, 0x8c?, 0xc003a17b6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0039cb9f8 sp=0xc0039cb9d8 pc=0x4907d6 runtime.selectgo(0xc0039cbb70, 0xc003a17b68, 0x4a59d2?, 0x0, 0xc46ca5?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0039cbb38 sp=0xc0039cb9f8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc011f8d440) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0039cbba0 sp=0xc0039cbb38 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00562dad0, {0x512dba0?, 0xc006483c20}, 0xc003a17ce8?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0039cbca8 sp=0xc0039cbba0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0039cbcd8 sp=0xc0039cbca8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011f8cfc0, 0xc003a17db0, 0xc003a17d80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc0039cbd30 sp=0xc0039cbcd8 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011f8cfc0, {0x512dba0?, 0xc006483c20?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0039cbde0 sp=0xc0039cbd30 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc003642c80, {0x512dba0?, 0xc006483c20?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc0039cbe28 sp=0xc0039cbde0 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc015e94a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:132 +0x4c fp=0xc0039cbe58 sp=0xc0039cbe28 pc=0x1d4b00c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x628a900, 0xc015e94a40}, 0xc000c62700, {0x624a948, 0xc006483b90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:409 +0x5b fp=0xc0039cbee0 sp=0xc0039cbe58 pc=0x1d843fb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x624a948?, 0xc006483b90?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:424 +0x45 fp=0xc0039cbf30 sp=0xc0039cbee0 pc=0x1d84345 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0039cbfe0 sp=0xc0039cbf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0039cbfe8 sp=0xc0039cbfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 459189 [sleep]: runtime.gopark(0x4b8eff47fc7?, 0x400e90?, 0xd6?, 0x4c?, 0x80e5e8a331?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00cf55df0 sp=0xc00cf55dd0 pc=0x4907d6 time.Sleep(0x152068c6a) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00cf55e30 sp=0xc00cf55df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc014d5c2a0, 0x400e90, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00cf55e88 sp=0xc00cf55e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc014d5c2a0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00cf55ec8 sp=0xc00cf55e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00fe51500?, 0x33, 0xc00ddd7f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00cf55fa8 sp=0xc00cf55ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00cf55fe0 sp=0xc00cf55fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00cf55fe8 sp=0xc00cf55fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 477702 [sleep]: runtime.gopark(0x4b8f0bb3479?, 0x400ea6?, 0x54?, 0x90?, 0x80e7c5e6a4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003a75df0 sp=0xc003a75dd0 pc=0x4907d6 time.Sleep(0x1531d1b6a) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003a75e30 sp=0xc003a75df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc010a821e0, 0x400ea6, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003a75e88 sp=0xc003a75e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc010a821e0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003a75ec8 sp=0xc003a75e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00b3c7fd0?, {0xc00798d500?, 0x41, 0xc0033488a0?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003a75fa8 sp=0xc003a75ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003a75fe0 sp=0xc003a75fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003a75fe8 sp=0xc003a75fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 120413 [select, 6 minutes]: runtime.gopark(0xc003bd9fb0?, 0x2?, 0x58?, 0x9f?, 0xc003bd9f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0049ede28 sp=0xc0049ede08 pc=0x4907d6 runtime.selectgo(0xc0049edfb0, 0xc003bd9f98, 0xc0038e3380?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0049edf68 sp=0xc0049ede28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0049edfe0 sp=0xc0049edf68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0049edfe8 sp=0xc0049edfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 451340 [sleep]: runtime.gopark(0x4b8862261e4?, 0x8019af?, 0x21?, 0xc0?, 0x805921827e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001367df0 sp=0xc001367dd0 pc=0x4907d6 time.Sleep(0x12f118a8d) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc001367e30 sp=0xc001367df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc007852f90, 0x8019af, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc001367e88 sp=0xc001367e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc007852f90, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc001367ec8 sp=0xc001367e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00129ac80?, 0x10, 0xc00f062e00?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc001367fa8 sp=0xc001367ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc001367fe0 sp=0xc001367fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001367fe8 sp=0xc001367fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498491 [select]: runtime.gopark(0xc01138b208?, 0x2?, 0x48?, 0xaf?, 0xc01138b204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01138b090 sp=0xc01138b070 pc=0x4907d6 runtime.selectgo(0xc01138b208, 0xc01138b200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01138b1d0 sp=0xc01138b090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc058a58a20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc01138b238 sp=0xc01138b1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030827130, {0x51b9a20?, 0xc0533f6a00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc01138b340 sp=0xc01138b238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc01138b370 sp=0xc01138b340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc058a586c0, 0xc01138b448, 0xc01138b418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc01138b3c8 sp=0xc01138b370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc058a586c0, {0x51b9a20?, 0xc0533f6a00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc01138b478 sp=0xc01138b3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00aeb34d0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc017227b30}, {0x51b9a20, 0xc0533f6a00}, 0x0?, {0xc00a393c00, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc01138b4e0 sp=0xc01138b478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00aeb34d0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc017227b30}, {0x51b9a20, 0xc0533f6a00}, 0x73c0685d53924601?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc01138b5d0 sp=0xc01138b4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00aeb34d0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc017227b30?}, {0x51b9a20?, 0xc0533f6a00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc01138b670 sp=0xc01138b5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0432774d0, {0x624a948, 0xc00aeb34d0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc01138b6f0 sp=0xc01138b670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0432774d0?}}, {0x624a948, 0xc00aeb34d0}, 0xc017227b30, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc01138b7c8 sp=0xc01138b6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc00aeb34d0?}, 0x624a948?, {0x0?, 0xc005751d20?, 0xc00c88d928?}) :1 +0x65 fp=0xc01138b818 sp=0xc01138b7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00df7bda0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc01138bab8 sp=0xc01138b818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00df7bda0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50c7348, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc01138bbf0 sp=0xc01138bab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50c7348, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc01138c338 sp=0xc01138bbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50c7348, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc01138cc38 sp=0xc01138c338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50c7348, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc01138d4a0 sp=0xc01138cc38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e50c7348, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc01138d8c0 sp=0xc01138d4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc01138d9f8 sp=0xc01138d8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00564de60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc01138dc00 sp=0xc01138d9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00564de60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc01138df30 sp=0xc01138dc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01138dfe0 sp=0xc01138df30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01138dfe8 sp=0xc01138dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49110 [runnable]: runtime.gopark(0x487350?, 0xb?, 0x0?, 0x0?, 0x74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0049f1aa0 sp=0xc0049f1a80 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x534825?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc0049f1ad8 sp=0xc0049f1aa0 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcbd168, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc0049f1af8 sp=0xc0049f1ad8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc011930080?, 0xc007e42000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0049f1b20 sp=0xc0049f1af8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011930080, {0xc007e42000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc0049f1ba0 sp=0xc0049f1b20 pc=0x535efa net.(*netFD).Read(0xc011930080, {0xc007e42000?, 0x459e71?, 0x4d964e0?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc0049f1be8 sp=0xc0049f1ba0 pc=0x7a20c9 net.(*conn).Read(0xc013f35bf0, {0xc007e42000?, 0x1c0?, 0xc0049f1c98?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc0049f1c30 sp=0xc0049f1be8 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00d92e2a0, {0xc007e42000, 0x8000, 0x8000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc0049f1c78 sp=0xc0049f1c30 pc=0x1641f96 bufio.(*Reader).Read(0xc00d92e300, {0xc004cd4580, 0x9, 0x0?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc0049f1cb0 sp=0xc0049f1c78 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc00d92e300}, {0xc004cd4580, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc0049f1cf8 sp=0xc0049f1cb0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004cd4580?, 0x9?, 0x0?}, {0x62052c0?, 0xc00d92e300?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc0049f1d48 sp=0xc0049f1cf8 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc004cd4540) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc0049f1df8 sp=0xc0049f1d48 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0074271e0, 0x6a4d4e6c6a49554c?, 0x574c7879504a7943?) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0x174 fp=0xc0049f1f18 sp=0xc0049f1df8 pc=0xbf09b4 google.golang.org/grpc.(*Server).serveStreams(0xc00296b6c0, {0x6294e00?, 0xc0074271e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 fp=0xc0049f1f80 sp=0xc0049f1f18 pc=0xc1d702 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 fp=0xc0049f1fe0 sp=0xc0049f1f80 pc=0xc1cec6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0049f1fe8 sp=0xc0049f1fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 498439 [select]: runtime.gopark(0xc011f15808?, 0x3?, 0x4?, 0x0?, 0xc011f154d2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc011f15258 sp=0xc011f15238 pc=0x4907d6 runtime.selectgo(0xc011f15808, 0xc011f154cc, 0x17600a76473815be?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc011f15398 sp=0xc011f15258 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003861800, {0x624a948, 0xc018c53ad0}, 0xc0054231d0, 0xc004ef7360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc011f15b08 sp=0xc011f15398 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc018c53ad0}, 0xc0054231d0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc011f15e28 sp=0xc011f15b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a764676a3b3, 0x0, 0x0}, 0x0, {0x17600a76e1918eac, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc011f161f0 sp=0xc011f15e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a764676a3b3, 0x0, 0x0}, 0x0, {0x17600a76e1918eac, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc011f16b78 sp=0xc011f161f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e1918eac, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc011f16dc0 sp=0xc011f16b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc018c538f0}, {0xc011f17470?}, 0xc0054230e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc011f17430 sp=0xc011f16dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc018c536b0}, 0xc0054230e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc011f174d8 sp=0xc011f17430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc018c536b0}, {0x52f0080?, 0xc0054230e0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc011f17518 sp=0xc011f174d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc018c536b0}, {0x52f0080, 0xc0054230e0}, 0xc010c707a0, 0xc013dcd050) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc011f17848 sp=0xc011f17518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c536b0?}, {0x52f0080?, 0xc0054230e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc011f17888 sp=0xc011f17848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc018c536b0}, {0x52f0080, 0xc0054230e0}, 0xc011f17920?, 0xc008d8cd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc011f178b8 sp=0xc011f17888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c536b0?}, {0x52f0080?, 0xc0054230e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc011f178f8 sp=0xc011f178b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc018c536b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc011f17930 sp=0xc011f178f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc018c536b0}, {0x203004?, 0x40?}, 0xc011f179e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc011f179a8 sp=0xc011f17930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc018c536b0?}, {0x52f0080?, 0xc0054230e0?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc011f17a20 sp=0xc011f179a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc018c536b0?}, {0x52f0080?, 0xc0054230e0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc011f17a60 sp=0xc011f17a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc018c536b0}, {0x52f0080, 0xc0054230e0}, 0xc010c707a0, 0xc013dcd050) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc011f17ab0 sp=0xc011f17a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc018c536b0}, 0xc00d1de9c0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc011f17b08 sp=0xc011f17ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc008149320, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc011f17e48 sp=0xc011f17b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc008149320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc011f17f68 sp=0xc011f17e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc011f17fe0 sp=0xc011f17f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc011f17fe8 sp=0xc011f17fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498317 [select]: runtime.gopark(0xc00a449c18?, 0x3?, 0x80?, 0xd?, 0xc00a449b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a4498f0 sp=0xc00a4498d0 pc=0x4907d6 runtime.selectgo(0xc00a449c18, 0xc00a449b34, 0xc0081520c0?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a449a30 sp=0xc00a4498f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc0034e9320}, {0x6221560, 0xc024ff7b00}, {0x46?, {0xc00a449e01?, 0xa449e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00a449da0 sp=0xc00a449a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc0034e9320?}, 0x164, 0xc024ff7b00, 0x5?, 0xc0120e69f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00a449e70 sp=0xc00a449da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc0034e9320}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00a449f30 sp=0xc00a449e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a449fe0 sp=0xc00a449f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a449fe8 sp=0xc00a449fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 118156 [select]: runtime.gopark(0xc001b65f00?, 0x2?, 0x76?, 0x0?, 0xc001b65ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001b65d60 sp=0xc001b65d40 pc=0x4907d6 runtime.selectgo(0xc001b65f00, 0xc001b65ed0, 0x7b55a5?, 0x0, 0xc011550000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001b65ea0 sp=0xc001b65d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc011b76820, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc001b65f30 sp=0xc001b65ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc011c24540) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc001b65f88 sp=0xc001b65f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc001b65fe0 sp=0xc001b65f88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001b65fe8 sp=0xc001b65fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 341845 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc014f73640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc014f735d0 sp=0xc014f735b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00e8a52d0, 0x345) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc014f73618 sp=0xc014f735d0 pc=0x4be30c sync.(*Cond).Wait(0xc00c2d5800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc014f73650 sp=0xc014f73618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00c2d5e68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc014f73708 sp=0xc014f73650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003bbb300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc014f73cb8 sp=0xc014f73708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003bbb300, {0x624a8a0, 0xc00e8a55c0}, 0x4150c5a880000000?, 0xc008421dc0?, 0xc00fc00b60?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc014f73d70 sp=0xc014f73cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00e8a55c0?}, {0xc0059fa000?}, 0x3?, 0xc0003faa98?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc014f73e20 sp=0xc014f73d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc014f73fe0 sp=0xc014f73e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc014f73fe8 sp=0xc014f73fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 488343 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc00ad9d640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b57f5d0 sp=0xc00b57f5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00a28fc50, 0x21) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc00b57f618 sp=0xc00b57f5d0 pc=0x4be30c sync.(*Cond).Wait(0xc0077e6000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc00b57f650 sp=0xc00b57f618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0077e6668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc00b57f708 sp=0xc00b57f650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc007c0a600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc00b57fcb8 sp=0xc00b57f708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc007c0a600, {0x624a8a0, 0xc00a28fc80}, 0x41473af080000000?, 0xc003a79dc0?, 0xc0174ebc50?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc00b57fd70 sp=0xc00b57fcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00a28fc80?}, {0xc0059fa000?}, 0x3?, 0xc000ab7500?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc00b57fe20 sp=0xc00b57fd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc00b57ffe0 sp=0xc00b57fe20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b57ffe8 sp=0xc00b57ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 498160 [chan receive]: runtime.gopark(0xc0043b12d0?, 0x49b1380?, 0x20?, 0x24?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0162d0400 sp=0xc0162d03e0 pc=0x4907d6 runtime.chanrecv(0xc00d431080, 0x0, 0x1) GOROOT/src/runtime/chan.go:583 +0x49b fp=0xc0162d0490 sp=0xc0162d0400 pc=0x45a91b runtime.chanrecv1(0x19bae6b?, 0x0?) GOROOT/src/runtime/chan.go:442 +0x18 fp=0xc0162d04b8 sp=0xc0162d0490 pc=0x45a418 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:543 +0x3a fp=0xc0162d04e0 sp=0xc0162d04b8 pc=0x19baeda runtime.deferreturn() GOROOT/src/runtime/panic.go:476 +0x33 fp=0xc0162d0520 sp=0xc0162d04e0 pc=0x48c7b3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc00189b7a0, {0x624a948, 0xc01502a090}, 0xc001821e00, 0xc0019cea80, 0xc010f0cfa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:606 +0x125b fp=0xc0162d0e00 sp=0xc0162d0520 pc=0x19b9cdb github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x624a948?, 0xc01502a090?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:515 +0x52 fp=0xc0162d0e50 sp=0xc0162d0e00 pc=0x19b8912 runtime/pprof.Do({0x624a948?, 0xc01502a000?}, {{0xc0043b1280?, 0x624a948?, 0xc01502a000?}}, 0xc00db7d0f8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0162d0ec0 sp=0xc0162d0e50 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc00189b7a0, {0x624a948, 0xc01502a000}, 0xc001821e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:514 +0xb25 fp=0xc0162d1188 sp=0xc0162d0ec0 pc=0x19b87c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc003b54ff0, {0x624a948, 0xc01502a000}, {0x0, {0x17600a76d8800bb0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:348 +0x9f fp=0xc0162d11c8 sp=0xc0162d1188 pc=0x1cbd8bf github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc003b54ff0, {0x624a948, 0xc01502a000}, 0xc012b15400, 0xc00db7d768?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc0162d1638 sp=0xc0162d11c8 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc01502a000}, 0xc00db7d820?, {0x0, {0x17600a76d8800bb0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0162d1778 sp=0xc0162d1638 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00130a400, {0x624a948, 0xc01502a000}, 0xc00561e2d0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0162d1a98 sp=0xc0162d1778 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00130a400, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc0162d1e60 sp=0xc0162d1a98 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0162d27e8 sp=0xc0162d1e60 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0162d2a30 sp=0xc0162d27e8 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc005d5b470}, {0x0?}, 0xc00561e1e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0162d30a0 sp=0xc0162d2a30 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc005d5b110}, 0xc00561e1e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0162d3148 sp=0xc0162d30a0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc005d5b110?}, {0x52f0080?, 0xc00561e1e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc0162d3178 sp=0xc0162d3148 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc005d5b110}, {0x52f0080, 0xc00561e1e0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0162d34a8 sp=0xc0162d3178 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc005d5b110?}, {0x52f0080?, 0xc00561e1e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0162d34e8 sp=0xc0162d34a8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc005d5b110}, {0x52f0080, 0xc00561e1e0}, 0xc00db7f580?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0162d3518 sp=0xc0162d34e8 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc005d5b110?}, {0x52f0080?, 0xc00561e1e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0162d3558 sp=0xc0162d3518 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc005d5b110?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0162d3590 sp=0xc0162d3558 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc005d5b110}, {0x1000000000018?, 0x7f27644c0050?}, 0xc00db7f648) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0162d3608 sp=0xc0162d3590 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc005d5b110?}, {0x52f0080?, 0xc00561e1e0?}, 0x40?, 0xc002100400?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0162d3680 sp=0xc0162d3608 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc005d5b110?}, {0x52f0080?, 0xc00561e1e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0162d36c0 sp=0xc0162d3680 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc005d5b110?}, {0xc005d5b110?, 0x4?}, {0x52f0080?, 0xc00561e1e0?}, {0x51b9a20?, 0xc012a7ad80}, 0x203001?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc0162d3700 sp=0xc0162d36c0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc005d5b110}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00561e1e0}, {0x51b9a20, 0xc012a7ad80}, 0xc00db7f878?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc0162d37f0 sp=0xc0162d3700 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc005d5b110}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00561e1e0}, {0x51b9a20, 0xc012a7ad80}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc0162d3888 sp=0xc0162d37f0 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc005d5aff0}, 0xc00561e1e0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc0162d3908 sp=0xc0162d3888 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x1?, {0x624a948?, 0xc005d5aff0?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc0162d3990 sp=0xc0162d3908 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc011d627e0, {_, _}, _, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0162d3c30 sp=0xc0162d3990 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc011d627e0, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0162d3d68 sp=0xc0162d3c30 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0162d44b0 sp=0xc0162d3d68 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0162d4db0 sp=0xc0162d44b0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0162d5618 sp=0xc0162d4db0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x17600a76d8801a0d, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0162d5a38 sp=0xc0162d5618 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0162d5b70 sp=0xc0162d5a38 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc0162d5c98 sp=0xc0162d5b70 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0xc03cf42380?, {_, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc0162d5db8 sp=0xc0162d5c98 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76d8800bb0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc0162d5ec8 sp=0xc0162d5db8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc044dff530}, 0x0?, 0xc0115c6c00) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc0162d60c0 sp=0xc0162d5ec8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc044dff530}, 0x19?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc0162d6108 sp=0xc0162d60c0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0005593b0, {0x624a948, 0xc044dff530}, 0xc00db82678, {{0x17600a76d87ff75f, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:395 +0x6fb fp=0xc0162d6578 sp=0xc0162d6108 pc=0x1ca4d3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0xc011581c20?, {0x624a948, 0xc044dff530}, 0xc014f09f18, {{0x17600a76d87ff75f, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x145 fp=0xc0162d6788 sp=0xc0162d6578 pc=0x1ca4505 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc003b55040, {0x624a948, 0xc044dff530}, {0xc006a486c0, {0x17600a76d4010d43, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:519 +0x596 fp=0xc0162d6d40 sp=0xc0162d6788 pc=0x1cd0d36 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc044dff530?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc0162d6e70 sp=0xc0162d6d40 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc003b55040, {0x624a948, 0xc044dff530}, {0xc006a486c0, {0x17600a76d4010d43, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc0162d7558 sp=0xc0162d6e70 pc=0x1ccefa6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc003b54ff0, {0x624a948, 0xc044dff530}, 0xc004575e00, 0xc00db83568?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc0162d79c8 sp=0xc0162d7558 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc044dff530}, 0xc004575e00?, {0xc006a486c0, {0x17600a76d4010d43, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0162d7b08 sp=0xc0162d79c8 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00130a400, {0x624a948, 0xc044dff530}, 0xc021831b30, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0162d7e28 sp=0xc0162d7b08 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00130a400, {_, _}, {{{0x17600a76d4010d43, 0x0, 0x0}, 0x0, {0x17600a76d40d8454, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc0162d81f0 sp=0xc0162d7e28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76d4010d43, 0x0, 0x0}, 0x0, {0x17600a76d40d8454, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0162d8b78 sp=0xc0162d81f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d40d8454, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0162d8dc0 sp=0xc0162d8b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc044dff4d0}, {0x7176634546734c72?}, 0xc021831a40) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0162d9430 sp=0xc0162d8dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc044dff410}, 0xc021831a40) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0162d94d8 sp=0xc0162d9430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc044dff410}, {0x52f0080?, 0xc021831a40}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc0162d9518 sp=0xc0162d94d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc044dff410}, {0x52f0080, 0xc021831a40}, 0xc006b778e0, 0xc006eaafd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0162d9848 sp=0xc0162d9518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc044dff410?}, {0x52f0080?, 0xc021831a40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc0162d9888 sp=0xc0162d9848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc044dff410}, {0x52f0080, 0xc021831a40}, 0xc00db85920?, 0xc00c96e900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0162d98b8 sp=0xc0162d9888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc044dff410?}, {0x52f0080?, 0xc021831a40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0162d98f8 sp=0xc0162d98b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc044dff410?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0162d9930 sp=0xc0162d98f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc044dff410}, {0x460d7e?, 0x40?}, 0xc00db859e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0162d99a8 sp=0xc0162d9930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc044dff410?}, {0x52f0080?, 0xc021831a40?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0162d9a20 sp=0xc0162d99a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc044dff410?}, {0x52f0080?, 0xc021831a40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0162d9a60 sp=0xc0162d9a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc044dff410}, {0x52f0080, 0xc021831a40}, 0xc006b778e0, 0xc006eaafd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc0162d9ab0 sp=0xc0162d9a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc044dff410}, 0xc00cc85980, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc0162d9b08 sp=0xc0162d9ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc012717320, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc0162d9e48 sp=0xc0162d9b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc012717320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc0162d9f68 sp=0xc0162d9e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc0162d9fe0 sp=0xc0162d9f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0162d9fe8 sp=0xc0162d9fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 49086 [select, 7 minutes]: runtime.gopark(0xc0079d3b70?, 0x2?, 0x0?, 0x0?, 0xc0079d3b6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0085b99f8 sp=0xc0085b99d8 pc=0x4907d6 runtime.selectgo(0xc0085b9b70, 0xc0079d3b68, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0085b9b38 sp=0xc0085b99f8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc007f6c900) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0085b9ba0 sp=0xc0085b9b38 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00562db80, {0x512dba0?, 0xc00753f050}, 0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0085b9ca8 sp=0xc0085b9ba0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0085b9cd8 sp=0xc0085b9ca8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc011f8dd40, 0xc0079d3db0, 0xc0079d3d80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc0085b9d30 sp=0xc0085b9cd8 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc011f8dd40, {0x512dba0?, 0xc00753f050?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0085b9de0 sp=0xc0085b9d30 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc003643980, {0x512dba0?, 0xc00753f050?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc0085b9e28 sp=0xc0085b9de0 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc015e94d70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:132 +0x4c fp=0xc0085b9e58 sp=0xc0085b9e28 pc=0x1d4b00c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x628a900, 0xc015e94d70}, 0xc000c62700, {0x624a948, 0xc00753eff0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:409 +0x5b fp=0xc0085b9ee0 sp=0xc0085b9e58 pc=0x1d843fb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x624a948?, 0xc00753eff0?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:424 +0x45 fp=0xc0085b9f30 sp=0xc0085b9ee0 pc=0x1d84345 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0085b9fe0 sp=0xc0085b9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0085b9fe8 sp=0xc0085b9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498499 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 fp=0xc00670ffe0 sp=0xc00670ffd8 pc=0xc1da20 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00670ffe8 sp=0xc00670ffe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 115789 [select, 6 minutes]: runtime.gopark(0xc004c94fb0?, 0x2?, 0x0?, 0x0?, 0xc004c94f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004c94e18 sp=0xc004c94df8 pc=0x4907d6 runtime.selectgo(0xc004c94fb0, 0xc004c94f90, 0xc004c94fd0?, 0x0, 0xc0035c51e0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004c94f58 sp=0xc004c94e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc004c94fe0 sp=0xc004c94f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004c94fe8 sp=0xc004c94fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 362899 [sleep]: runtime.gopark(0x4b8b4fd4941?, 0x2060d6?, 0x3?, 0xac?, 0x808f573865?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0085bfdf0 sp=0xc0085bfdd0 pc=0x4907d6 time.Sleep(0x1366c57f5) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc0085bfe30 sp=0xc0085bfdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc010a4aae0, 0x2060d6, {0x4618e7?, 0x203004?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc0085bfe88 sp=0xc0085bfe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc010a4aae0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc0085bfec8 sp=0xc0085bfe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00c554000?, 0x31, 0xc006644f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc0085bffa8 sp=0xc0085bfec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc0085bffe0 sp=0xc0085bffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0085bffe8 sp=0xc0085bffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 256409 [sleep]: runtime.gopark(0x4b85d568b84?, 0x20091a?, 0xae?, 0xfc?, 0x8022f4f50c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00150edf0 sp=0xc00150edd0 pc=0x4907d6 time.Sleep(0x121b0d3a8) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00150ee30 sp=0xc00150edf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc017fa5c20, 0x20091a, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00150ee88 sp=0xc00150ee30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc017fa5c20, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00150eec8 sp=0xc00150ee88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc015367f80?, {0xc00bb3ea00?, 0x6d, 0xc015367f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00150efa8 sp=0xc00150eec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00150efe0 sp=0xc00150efa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00150efe8 sp=0xc00150efe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 404694 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x3ef?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c284f58 sp=0xc00c284f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7720?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00c284f90 sp=0xc00c284f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7638, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00c284fb0 sp=0xc00c284f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc03f268580?, 0xc002ffc000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00c284fd8 sp=0xc00c284fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc03f268580, {0xc002ffc000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00c285058 sp=0xc00c284fd8 pc=0x535efa net.(*netFD).Read(0xc03f268580, {0xc002ffc000?, 0x7b5a68?, 0xc03f268580?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00c2850a0 sp=0xc00c285058 pc=0x7a20c9 net.(*conn).Read(0xc0014fafe0, {0xc002ffc000?, 0x164290f?, 0xc0014fafe0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00c2850e8 sp=0xc00c2850a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc008378de0, {0xc002ffc000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00c285130 sp=0xc00c2850e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc059d12360, {0xc002ffc000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00c2851d0 sp=0xc00c285130 pc=0x370f896 bufio.(*Reader).fill(0xc005caa490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00c285208 sp=0xc00c2851d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc005caa490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00c285228 sp=0xc00c285208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc005caaa68?, {0x62295c8, 0xc005caa490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00c285260 sp=0xc00c285228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc005caa400, 0xc0125103d0, 0xc00c285442, 0xc00c285444, {0x623b718, 0xc011c06480}, 0xc0059fa000, 0xc014dfb1f4, 0xc00c285440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00c285398 sp=0xc00c285260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc005caa400, {0x624a948, 0xc0101b67b0}, 0xc0125103c0, 0xc0059fa000, 0xc0101b66f0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00c285768 sp=0xc00c285398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0101b67b0}, {0x628a538, 0xc008378de0}, {{{0xc034465009, 0x4}}, 0x1, 0xc0101b6720, 0xc0101b6750, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00c2858a8 sp=0xc00c285768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0101b67b0}, {0x628a538?, 0xc008378de0}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00c285e20 sp=0xc00c2858a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0101b6630?}, {0x628a538, 0xc008378de0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00c285eb0 sp=0xc00c285e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0101b6630}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00c285f30 sp=0xc00c285eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00c285fe0 sp=0xc00c285f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c285fe8 sp=0xc00c285fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 122119 [select]: runtime.gopark(0xc003991a10?, 0x2?, 0xff?, 0x28?, 0xc0039919f4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003991878 sp=0xc003991858 pc=0x4907d6 runtime.selectgo(0xc003991a10, 0xc0039919f0, 0xc000580000?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0039919b8 sp=0xc003991878 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00e666730, {0xc01314add8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc003991a40 sp=0xc0039919b8 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00e666730, {0xc01314add8?, 0xc004615860?, 0xc003991a88?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc003991a80 sp=0xc003991a40 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc012b40e40, {0xc01314add8?, 0xc003991b00?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc003991ac8 sp=0xc003991a80 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc012b40e40}, {0xc01314add8, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc003991b10 sp=0xc003991ac8 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0142dad80, {0xc01314add8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc003991b50 sp=0xc003991b10 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc01314adc8, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc003991bd0 sp=0xc003991b50 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x1e63756?, 0xc0142dad80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc003991c68 sp=0xc003991bd0 pc=0xc19546 google.golang.org/grpc.recv(0x1d1a426?, {0x7f275f301038, 0xc0000af170}, 0x2?, {0x0?, 0x0?}, {0x512ca20, 0xc013666a20}, 0x203004?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc003991ce0 sp=0xc003991c68 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc012632340, {0x512ca20?, 0xc013666a20}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc003991e10 sp=0xc003991ce0 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x9172430?, {0x512ca20?, 0xc013666a20?}) :1 +0x34 fp=0xc003991e38 sp=0xc003991e10 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc0016c5790) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:363 +0x4c fp=0xc003991e68 sp=0xc003991e38 pc=0x16c594c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x624a948, 0xc012b41020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 fp=0xc003991f30 sp=0xc003991e68 pc=0x1d1a762 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc003991fe0 sp=0xc003991f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003991fe8 sp=0xc003991fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 118154 [select]: runtime.gopark(0xc00da28f48?, 0x2?, 0x6?, 0x0?, 0xc00da28ef4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00da28d50 sp=0xc00da28d30 pc=0x4907d6 runtime.selectgo(0xc00da28f48, 0xc00da28ef0, 0x0?, 0x0, 0x3?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00da28e90 sp=0xc00da28d50 pc=0x4a071c google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc011c20f00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x165 fp=0xc00da28fc8 sp=0xc00da28e90 pc=0xbeb085 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x26 fp=0xc00da28fe0 sp=0xc00da28fc8 pc=0xbe0906 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00da28fe8 sp=0xc00da28fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x197c goroutine 115793 [select, 6 minutes]: runtime.gopark(0xc004ee5e58?, 0x2?, 0x3?, 0x30?, 0xc004ee5dfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004ee5c80 sp=0xc004ee5c60 pc=0x4907d6 runtime.selectgo(0xc004ee5e58, 0xc004ee5df8, 0xc006c4ea80?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004ee5dc0 sp=0xc004ee5c80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc006c4ea80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc004ee5f58 sp=0xc004ee5dc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc004ee5f78 sp=0xc004ee5f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc004ee5fe0 sp=0xc004ee5f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004ee5fe8 sp=0xc004ee5fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 498421 [runnable]: runtime.gopark(0xae1b649ba69?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fb839a0 sp=0xc00fb83980 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00fb83a08 sp=0xc00fb839a0 pc=0x4a17de sync.runtime_SemacquireMutex(0xc00fb83aa0?, 0x8f?, 0xc011d63560?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00fb83a38 sp=0xc00fb83a08 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x9c16980?, 0xc00fb83ad0?, 0x4be125?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00fb83a90 sp=0xc00fb83a38 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10a362f?, 0xc0005ee3e0?, 0xc00fb83b20?, 0x10b86d8?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00fb83ad8 sp=0xc00fb83a90 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004e6b340, {0xc00fb83c18?, 0x4ccece?}, 0x0, 0x0, 0xc006720810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00fb83bd8 sp=0xc00fb83ad8 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00586e000, 0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc00fb83c28 sp=0xc00fb83bd8 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc006720810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc00fb83cb0 sp=0xc00fb83c28 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc002941970, 0xfa17a8?, 0xc006721740, {0x0?, 0xc006720810?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc00fb83de0 sp=0xc00fb83cb0 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x7f2581044798?, 0x0?, 0xc006ec9400?, {0x0?, 0xc006720810?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc00fb83e30 sp=0xc00fb83de0 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc0067216f0?, 0xc005abd080?, {0x0?, 0xc006720810?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc00fb83e78 sp=0xc00fb83e30 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0067216f0, 0xc005abd080, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc00fb83ed8 sp=0xc00fb83e78 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0067216f0, {0xc006ec9400, 0xd, 0x20}, {0xc010cb91c0, 0xd, 0xd}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc00fb83f28 sp=0xc00fb83ed8 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc006720a20, {0xc010cb91c0?, 0x49e9380?, 0x1?}, 0xc00fb84120?, 0xed?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00fb84058 sp=0xc00fb83f28 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc006720a20, {0xc006ec9400?, 0xd?, 0xc005ce9400?}, {0xc010cb91c0?, 0xc005ce9400?, 0xc?}, 0x50?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00fb84098 sp=0xc00fb84058 pc=0x115285d github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc006720500, {0xc010cb91c0, 0xd, 0xd}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00fb84130 sp=0xc00fb84098 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc006acc2d8, {{0xc005ce9400, 0xc, 0x50}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:373 +0x147 fp=0xc00fb84188 sp=0xc00fb84130 pc=0x1863127 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0116b0f00, {{0xc0040f15e8, 0x6, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:520 +0x4d5 fp=0xc00fb84248 sp=0xc00fb84188 pc=0x180d135 github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x62b4f18, 0xc0116b0f00}, {{0xc0040f15e8, 0x6, 0x8}, {0x0, 0x0, 0x0}}, 0xc005b9e360) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1129 +0xc3 fp=0xc00fb84698 sp=0xc00fb84248 pc=0x1817843 github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x624a948, 0xc0009f9d70}, {0x62b8438?, 0xc006acc000?}, {0x62b4f18, 0xc0116b0f00}, 0xc002e2be00, {0xc0040f15e8, 0x6, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1650 +0x165 fp=0xc00fb853a8 sp=0xc00fb84698 pc=0x181aa25 github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x624a948, 0xc0009f9d70}, {0x62b8438, 0xc006acc000}, {0x62b4f18, 0xc0116b0f00}, 0x9?, {0xc0040f15e8, 0x6, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1414 +0x1cd fp=0xc00fb854b0 sp=0xc00fb853a8 pc=0x18197cd github.com/cockroachdb/cockroach/pkg/storage.MVCCPut({0x624a948, 0xc0009f9d70}, {0x7f275f2b7158?, 0xc006acc000}, 0xc002e2be00, {0xc0040f15e8, 0x6, 0x8}, {0x17600a76df8cb55e, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1333 +0x2a5 fp=0xc00fb855c0 sp=0xc00fb854b0 pc=0x1819005 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Put({_, _}, {_, _}, {{0x62cc4b0, 0xc016716540}, {{0x17600a76df8cb55e, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_put.go:59 +0x311 fp=0xc00fb85bf0 sp=0xc00fb855c0 pc=0x1cfa251 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76df8cb55e, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc00fb86480 sp=0xc00fb85bf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00a5afa40, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00fb882f0 sp=0xc00fb86480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00e9e37c0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00fb88940 sp=0xc00fb882f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00e9e37c0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00fb88e18 sp=0xc00fb88940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76fd5a1a5e, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00fb89c98 sp=0xc00fb88e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc002100400?, {0x624a948, 0xc0009f9d70}, {0xc0040f15f0, 0x8}, 0xc00a5afa40, 0x8?, 0x0?, {{0x17600a76fd5a1a5e, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00fb8a918 sp=0xc00fb89c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc0009f9d70}, {0xc0040f15f0, 0x8}, 0xc00a5afa40, 0x400000004?, 0xc00fb8b968, {{0x17600a76fd5a1a5e, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00fb8a9d0 sp=0xc00fb8a918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc006f10000, {0x624a948, 0xc0009f9d70}, 0xc00a5afa40, 0xc00e9e37c0, 0xc00fb8b968, {{0x17600a76fd5a1a5e, 0x0, 0x0}, {0x17600a76e05e6005, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00fb8b398 sp=0xc00fb8a9d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc0009f9d70}, 0xc00a5afa40, 0xc00e9e37c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00fb8bb08 sp=0xc00fb8b398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc0009f9d70}, 0xc00a5afa40, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00fb8be28 sp=0xc00fb8bb08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76df8cb55e, 0x0, 0x0}, 0x0, {0x17600a76e2b1199b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00fb8c1f0 sp=0xc00fb8be28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76df8cb55e, 0x0, 0x0}, 0x0, {0x17600a76e2b1199b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00fb8cb78 sp=0xc00fb8c1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xff3d15f6d74568b4?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2b1199b, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00fb8cdc0 sp=0xc00fb8cb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc003497d10}, {0xc012dcf188?}, 0xc00a5af950) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00fb8d430 sp=0xc00fb8cdc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc003497950}, 0xc00a5af950) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00fb8d4d8 sp=0xc00fb8d430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc003497950}, {0x52f0080?, 0xc00a5af950}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00fb8d518 sp=0xc00fb8d4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc003497950}, {0x52f0080, 0xc00a5af950}, 0xc010910060, 0xc01285e6a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00fb8d848 sp=0xc00fb8d518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc003497950?}, {0x52f0080?, 0xc00a5af950?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00fb8d888 sp=0xc00fb8d848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc003497950}, {0x52f0080, 0xc00a5af950}, 0xc00fb8d920?, 0xc005e4e480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00fb8d8b8 sp=0xc00fb8d888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc003497950?}, {0x52f0080?, 0xc00a5af950?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fb8d8f8 sp=0xc00fb8d8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc003497950?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00fb8d930 sp=0xc00fb8d8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc003497950}, {0x203004?, 0x40?}, 0xc00fb8d9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00fb8d9a8 sp=0xc00fb8d930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc003497950?}, {0x52f0080?, 0xc00a5af950?}, 0x5003b00?, 0x7f278cfaa501?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00fb8da20 sp=0xc00fb8d9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc003497950?}, {0x52f0080?, 0xc00a5af950?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fb8da60 sp=0xc00fb8da20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc003497950}, {0x52f0080, 0xc00a5af950}, 0xc010910060, 0xc01285e6a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00fb8dab0 sp=0xc00fb8da60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc003497950}, 0xc015e00c60, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00fb8db08 sp=0xc00fb8dab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00cc6a000, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00fb8de48 sp=0xc00fb8db08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00cc6a000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00fb8df68 sp=0xc00fb8de48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00fb8dfe0 sp=0xc00fb8df68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fb8dfe8 sp=0xc00fb8dfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 497153 [sleep]: runtime.gopark(0x4b8ce916def?, 0x1002eb2?, 0x21?, 0x6c?, 0x80c376f880?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00150bdf0 sp=0xc00150bdd0 pc=0x4907d6 time.Sleep(0x150f7f805) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00150be30 sp=0xc00150bdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc007791dd0, 0x1002eb2, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00150be88 sp=0xc00150be30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc007791dd0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00150bec8 sp=0xc00150be88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc0125a8960?, 0x3, 0xc00853ffb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00150bfa8 sp=0xc00150bec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00150bfe0 sp=0xc00150bfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00150bfe8 sp=0xc00150bfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 49171 [select]: runtime.gopark(0xc0053e9a08?, 0x2?, 0xc8?, 0xe8?, 0xc0053e99ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0053e9870 sp=0xc0053e9850 pc=0x4907d6 runtime.selectgo(0xc0053e9a08, 0xc0053e99e8, 0x16e9e6c?, 0x0, 0xc007b1f817?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0053e99b0 sp=0xc0053e9870 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002cff400, {0xc005645870, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc0053e9a38 sp=0xc0053e99b0 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002cff400, {0xc005645870?, 0xc003293920?, 0xc0053e9a80?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc0053e9a78 sp=0xc0053e9a38 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc004bd7230, {0xc005645870?, 0xc0053e9af8?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc0053e9ac0 sp=0xc0053e9a78 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc004bd7230}, {0xc005645870, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc0053e9b08 sp=0xc0053e9ac0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc004960d80, {0xc005645870, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc0053e9b48 sp=0xc0053e9b08 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc005645860, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc0053e9bc8 sp=0xc0053e9b48 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x4908c0?, 0xc004960d80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc0053e9c60 sp=0xc0053e9bc8 pc=0xc19546 google.golang.org/grpc.recv(0xc0053e9d10?, {0x7f275f301038, 0xc0000af170}, 0x0?, {0x0?, 0x0?}, {0x512da60, 0xc004feb800}, 0xff?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc0053e9cd8 sp=0xc0053e9c60 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00a7fb450, {0x512da60?, 0xc004feb800}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc0053e9e08 sp=0xc0053e9cd8 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0xc01237a440?, {0x512da60?, 0xc004feb800?}) :1 +0x34 fp=0xc0053e9e30 sp=0xc0053e9e08 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00df462e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:263 +0x4c fp=0xc0053e9e60 sp=0xc0053e9e30 pc=0x1d4b66c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:91 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x628a958?, 0xc00df462e0}, 0xc000c62700, {0x624a948, 0xc004bd7470}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:289 +0xa6 fp=0xc0053e9ee0 sp=0xc0053e9e60 pc=0x1d82d66 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x624a948?, 0xc004bd7470?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:308 +0x45 fp=0xc0053e9f30 sp=0xc0053e9ee0 pc=0x1d82c65 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0053e9fe0 sp=0xc0053e9f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0053e9fe8 sp=0xc0053e9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115796 [select, 6 minutes]: runtime.gopark(0xc00dff3fb0?, 0x2?, 0x0?, 0x0?, 0xc00dff3f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dff3e18 sp=0xc00dff3df8 pc=0x4907d6 runtime.selectgo(0xc00dff3fb0, 0xc00dff3f90, 0xc00dff3fd0?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dff3f58 sp=0xc00dff3e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc00dff3fe0 sp=0xc00dff3f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dff3fe8 sp=0xc00dff3fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 498441 [runnable]: runtime.gopark(0xae1b68a5cd4?, 0x0?, 0x60?, 0xcc?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00b8ad3f8 sp=0xc00b8ad3d8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee65c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00b8ad460 sp=0xc00b8ad3f8 pc=0x4a17de sync.runtime_SemacquireMutex(0x0?, 0x0?, 0x0?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00b8ad490 sp=0xc00b8ad460 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee640, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00b8ad4e8 sp=0xc00b8ad490 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00b8ad530 sp=0xc00b8ad4e8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0033bd600, {0xc00b8ad670?, 0x4ccece?}, 0x0, 0x0, 0xc006412810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00b8ad630 sp=0xc00b8ad530 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00f618c00, 0xc0033bd600, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc00b8ad680 sp=0xc00b8ad630 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc0033bd600, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc006412810) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc00b8ad708 sp=0xc00b8ad680 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc002941810, 0x0?, 0xc006413e30, {0x0?, 0xc006412810?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc00b8ad838 sp=0xc00b8ad708 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x0?, 0x0?, 0x0?, {0x0?, 0xc006412810?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc00b8ad888 sp=0xc00b8ad838 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc006413de0?, 0xc0046c22c0?, {0x0?, 0xc006412810?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc00b8ad8d0 sp=0xc00b8ad888 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc006413de0, 0xc0046c22c0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc00b8ad930 sp=0xc00b8ad8d0 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc006413de0, {0xc00fec3dc0, 0x7, 0x40}, {0xc006a8a3a0, 0x7, 0x20}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc00b8ad980 sp=0xc00b8ad930 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc006412a20, {0xc006a8a3a0?, 0xc006ab8c28?, 0x0?}, 0x6?, 0x6?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00b8adab0 sp=0xc00b8ad980 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc006412a20, {0xc00fec3dc0?, 0x0?, 0x7?}, {0xc006a8a3a0?, 0x0?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00b8adaf0 sp=0xc00b8adab0 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc006412888, {0xc00fec3dc0?, 0x40?, 0xc006a8a3a0?}, {0xc006a8a3a0, 0x7, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00b8adb48 sp=0xc00b8adaf0 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc006412500, {0xc006a8a3a0, 0x7, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00b8adbe0 sp=0xc00b8adb48 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc006ab8c28, {{0xc0054c7ca0, 0x6, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00b8adc30 sp=0xc00b8adbe0 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekIntentGE(0xc0116a6a00, {0xc0054c7ca0, 0x6, 0x8}, {0xc, 0xd4, 0x5e, 0xf0, 0x0, 0x68, ...}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:540 +0x117 fp=0xc00b8add58 sp=0xc00b8adc30 pc=0x180d357 github.com/cockroachdb/cockroach/pkg/storage.MVCCResolveWriteIntent({0x624a948, 0xc0082e6cf0}, {0x7f275f2b7158?, 0xc006ab8c00?}, 0x0?, {{{0xc0054c7ca0, 0x6, 0x8}, {0x0, 0x0, ...}}, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3968 +0x12b fp=0xc00b8adeb0 sp=0xc00b8add58 pc=0x182dfab github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.resolveLocalLocks.func1(0xc00b8ae418, 0xc00b8ae468, 0xc00b8ae4a0, 0x122?, 0x1d3?, {0x624a948, 0xc0082e6cf0}, {0x7f275f2b7158, 0xc006ab8c00}, 0xc00c684140, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:557 +0x405 fp=0xc00b8ae378 sp=0xc00b8adeb0 pc=0x1ce98e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.resolveLocalLocks({0x624a948, 0xc0082e6cf0}, 0xc0029c3500, {0x7f275f2b7158, 0xc006ab8c00}, 0x0?, 0xc0426906c0, 0xc01104bc20, {0x62cc4b0, 0xc00d756f00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:610 +0x34f fp=0xc00b8ae570 sp=0xc00b8ae378 pc=0x1ce92cf github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.EndTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc00d756f00}, {{0x17600a76b7bbccde, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:412 +0xf6a fp=0xc00b8afbf0 sp=0xc00b8ae570 pc=0x1ce868a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76b7bbccde, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc00b8b0480 sp=0xc00b8afbf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc005423b30, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00b8b22f0 sp=0xc00b8b0480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0035ac780, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00b8b2940 sp=0xc00b8b22f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0035ac780, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00b8b2e18 sp=0xc00b8b2940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76d58931de, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00b8b3c98 sp=0xc00b8b2e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000580000?, {0x624a948, 0xc0082e6cf0}, {0xc0054c7db0, 0x8}, 0xc005423b30, 0x8?, 0x0?, {{0x17600a76d58931de, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00b8b4918 sp=0xc00b8b3c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc0082e6cf0}, {0xc0054c7db0, 0x8}, 0xc005423b30, 0x400000004?, 0xc00b8b5968, {{0x17600a76d58931de, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00b8b49d0 sp=0xc00b8b4918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc006f10000, {0x624a948, 0xc0082e6cf0}, 0xc005423b30, 0xc0035ac780, 0xc00b8b5968, {{0x17600a76d58931de, 0x0, 0x0}, {0x17600a76b9fe779a, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00b8b5398 sp=0xc00b8b49d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc0082e6cf0}, 0xc005423b30, 0xc0035ac780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00b8b5b08 sp=0xc00b8b5398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc0082e6cf0}, 0xc005423b30, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00b8b5e28 sp=0xc00b8b5b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76b7bbccde, 0x0, 0x0}, 0x0, {0x17600a76e2ed4f4c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00b8b61f0 sp=0xc00b8b5e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76b7bbccde, 0x0, 0x0}, 0x0, {0x17600a76e2ed4f4c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00b8b6b78 sp=0xc00b8b61f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc0025ecc80?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e2ed4f4c, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00b8b6dc0 sp=0xc00b8b6b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0082e6c30}, {0x459c3d?}, 0xc005423a40) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00b8b7430 sp=0xc00b8b6dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0082e6b10}, 0xc005423a40) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00b8b74d8 sp=0xc00b8b7430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc0082e6b10}, {0x52f0080?, 0xc005423a40}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00b8b7518 sp=0xc00b8b74d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0082e6b10}, {0x52f0080, 0xc005423a40}, 0xc010c70ba0, 0xc013dcd3c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00b8b7848 sp=0xc00b8b7518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0082e6b10?}, {0x52f0080?, 0xc005423a40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00b8b7888 sp=0xc00b8b7848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0082e6b10}, {0x52f0080, 0xc005423a40}, 0xc00b8b7920?, 0xc008d8d340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00b8b78b8 sp=0xc00b8b7888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0082e6b10?}, {0x52f0080?, 0xc005423a40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b8b78f8 sp=0xc00b8b78b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0082e6b10?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00b8b7930 sp=0xc00b8b78f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0082e6b10}, {0x203004?, 0x40?}, 0xc00b8b79e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00b8b79a8 sp=0xc00b8b7930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0082e6b10?}, {0x52f0080?, 0xc005423a40?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00b8b7a20 sp=0xc00b8b79a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0082e6b10?}, {0x52f0080?, 0xc005423a40?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00b8b7a60 sp=0xc00b8b7a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc0082e6b10}, {0x52f0080, 0xc005423a40}, 0xc010c70ba0, 0xc013dcd3c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00b8b7ab0 sp=0xc00b8b7a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc0082e6b10}, 0xc00d1df080, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00b8b7b08 sp=0xc00b8b7ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc006018c60, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00b8b7e48 sp=0xc00b8b7b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc00c483040}, 0xc006018c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00b8b7f68 sp=0xc00b8b7e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00b8b7fe0 sp=0xc00b8b7f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00b8b7fe8 sp=0xc00b8b7fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 49085 [select, 7 minutes]: runtime.gopark(0xc006457fb0?, 0x2?, 0x40?, 0xc2?, 0xc006457f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0106cae28 sp=0xc0106cae08 pc=0x4907d6 runtime.selectgo(0xc0106cafb0, 0xc006457f98, 0x1?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0106caf68 sp=0xc0106cae28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc0106cafe0 sp=0xc0106caf68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0106cafe8 sp=0xc0106cafe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 487697 [sleep]: runtime.gopark(0x4b8c6ea19e4?, 0x1002fa2?, 0xfc?, 0x2f?, 0x80b871bc4c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00853fdf0 sp=0xc00853fdd0 pc=0x4907d6 time.Sleep(0x14d9a0d96) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00853fe30 sp=0xc00853fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00eb28390, 0x1002fa2, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00853fe88 sp=0xc00853fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00eb28390, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00853fec8 sp=0xc00853fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc007fd6500?, 0x8, 0xc00ddd2f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00853ffa8 sp=0xc00853fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00853ffe0 sp=0xc00853ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00853ffe8 sp=0xc00853ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498136 [select]: runtime.gopark(0xc0162f0d80?, 0x7?, 0x5c?, 0x0?, 0xc0162f078a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0162f03e0 sp=0xc0162f03c0 pc=0x4907d6 runtime.selectgo(0xc0162f0d80, 0xc0162f077c, 0xc004959320?, 0x0, 0xc01164c040?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0162f0520 sp=0xc0162f03e0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc004959320, {0x624a948, 0xc01502bc50}, 0xc001ddb0e0, 0xc0019cef40, 0xc005f0b7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:569 +0x68f fp=0xc0162f0e00 sp=0xc0162f0520 pc=0x19b910f github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x624a948?, 0xc01502bc50?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:515 +0x52 fp=0xc0162f0e50 sp=0xc0162f0e00 pc=0x19b8912 runtime/pprof.Do({0x624a948?, 0xc01502bb30?}, {{0xc01164c000?, 0x624a948?, 0xc01502bb30?}}, 0xc0162f10f8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 fp=0xc0162f0ec0 sp=0xc0162f0e50 pc=0xfd3223 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc004959320, {0x624a948, 0xc01502bb30}, 0xc001ddb0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:514 +0xb25 fp=0xc0162f1188 sp=0xc0162f0ec0 pc=0x19b87c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc00b56cf00, {0x624a948, 0xc01502bb30}, {0x0, {0x17600a76d8942728, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:348 +0x9f fp=0xc0162f11c8 sp=0xc0162f1188 pc=0x1cbd8bf github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc01502bb30}, 0xc03f3e5ae0, 0xc010cded90?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc0162f1638 sp=0xc0162f11c8 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc01502bb30}, 0xc03f3e5ae0?, {0x0, {0x17600a76d8942728, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0162f1778 sp=0xc0162f1638 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc01502bb30}, 0xc00561f1d0, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0162f1a98 sp=0xc0162f1778 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc0162f1e60 sp=0xc0162f1a98 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0162f27e8 sp=0xc0162f1e60 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x0?, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0162f2a30 sp=0xc0162f27e8 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc01502ba70}, {0x0?}, 0xc00561f0e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0162f30a0 sp=0xc0162f2a30 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc01502b9b0}, 0xc00561f0e0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0162f3148 sp=0xc0162f30a0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc01502b9b0?}, {0x52f0080?, 0xc00561f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc0162f3178 sp=0xc0162f3148 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc01502b9b0}, {0x52f0080, 0xc00561f0e0}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0162f34a8 sp=0xc0162f3178 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc01502b9b0?}, {0x52f0080?, 0xc00561f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0162f34e8 sp=0xc0162f34a8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc01502b9b0}, {0x52f0080, 0xc00561f0e0}, 0xc00dcb5580?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0162f3518 sp=0xc0162f34e8 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc01502b9b0?}, {0x52f0080?, 0xc00561f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0162f3558 sp=0xc0162f3518 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc01502b9b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0162f3590 sp=0xc0162f3558 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc01502b9b0}, {0x1000000000018?, 0x7f2763840b78?}, 0xc00dcb5648) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0162f3608 sp=0xc0162f3590 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc01502b9b0?}, {0x52f0080?, 0xc00561f0e0?}, 0x40?, 0x9bffb20?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0162f3680 sp=0xc0162f3608 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc01502b9b0?}, {0x52f0080?, 0xc00561f0e0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc0162f36c0 sp=0xc0162f3680 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc01502b9b0?}, {0xc01502b9b0?, 0x4?}, {0x52f0080?, 0xc00561f0e0?}, {0x51b9a20?, 0xc012a7b800}, 0x203005?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc0162f3700 sp=0xc0162f36c0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc01502b9b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00561f0e0}, {0x51b9a20, 0xc012a7b800}, 0xc00dcb5878?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc0162f37f0 sp=0xc0162f3700 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc01502b9b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00561f0e0}, {0x51b9a20, 0xc012a7b800}, 0x4f056a?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc0162f3888 sp=0xc0162f37f0 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc01502b8c0}, 0xc00561f0e0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc0162f3908 sp=0xc0162f3888 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0x1?, {0x624a948?, 0xc01502b8c0?}, 0x0?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc0162f3990 sp=0xc0162f3908 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc013c07f80, {_, _}, _, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc0162f3c30 sp=0xc0162f3990 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc013c07f80, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc0162f3d68 sp=0xc0162f3c30 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc0162f44b0 sp=0xc0162f3d68 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc0162f4db0 sp=0xc0162f44b0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0162f5618 sp=0xc0162f4db0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x17600a76d8943080, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0162f5a38 sp=0xc0162f5618 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0162f5b70 sp=0xc0162f5a38 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc0162f5c98 sp=0xc0162f5b70 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0xc0157b0e00?, {_, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc0162f5db8 sp=0xc0162f5c98 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76d8942728, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc0162f5ec8 sp=0xc0162f5db8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc0078c5230}, 0x0?, 0xc008308b00) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc0162f60c0 sp=0xc0162f5ec8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc0078c5230}, 0x19?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc0162f6108 sp=0xc0162f60c0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0005593b0, {0x624a948, 0xc0078c5230}, 0xc00dcb8678, {{0x17600a76d89414b4, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:395 +0x6fb fp=0xc0162f6578 sp=0xc0162f6108 pc=0x1ca4d3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0xc00d484a20?, {0x624a948, 0xc0078c5230}, 0xc011a0df98, {{0x17600a76d89414b4, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x145 fp=0xc0162f6788 sp=0xc0162f6578 pc=0x1ca4505 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc00b56cf50, {0x624a948, 0xc0078c5230}, {0xc006fed9e0, {0x17600a76d39aa37c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:519 +0x596 fp=0xc0162f6d40 sp=0xc0162f6788 pc=0x1cd0d36 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x624a948?, 0xc0078c5230?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:378 +0x2bc fp=0xc0162f6e70 sp=0xc0162f6d40 pc=0x1cd065c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc00b56cf50, {0x624a948, 0xc0078c5230}, {0xc006fed9e0, {0x17600a76d39aa37c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:429 +0x4e6 fp=0xc0162f7558 sp=0xc0162f6e70 pc=0x1ccefa6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc0078c5230}, 0xc0122735e0, 0xc00dcb9568?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:329 +0x98f fp=0xc0162f79c8 sp=0xc0162f7558 pc=0x1cbd50f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc0078c5230}, 0xc0122735e0?, {0xc006fed9e0, {0x17600a76d39aa37c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc0162f7b08 sp=0xc0162f79c8 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc0078c5230}, 0xc004737e00, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc0162f7e28 sp=0xc0162f7b08 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76d39aa37c, 0x0, 0x0}, 0x0, {0x17600a76d454e1e2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc0162f81f0 sp=0xc0162f7e28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76d39aa37c, 0x0, 0x0}, 0x0, {0x17600a76d454e1e2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc0162f8b78 sp=0xc0162f81f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76d454e1e2, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc0162f8dc0 sp=0xc0162f8b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0078c51d0}, {0xc00dcbb460?}, 0xc004737d10) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc0162f9430 sp=0xc0162f8dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0078c5110}, 0xc004737d10) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc0162f94d8 sp=0xc0162f9430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc0078c5110}, {0x52f0080?, 0xc004737d10}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc0162f9518 sp=0xc0162f94d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0078c5110}, {0x52f0080, 0xc004737d10}, 0xc00f69f020, 0xc001e5b2f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc0162f9848 sp=0xc0162f9518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0078c5110?}, {0x52f0080?, 0xc004737d10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc0162f9888 sp=0xc0162f9848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0078c5110}, {0x52f0080, 0xc004737d10}, 0xc00dcbb920?, 0xc00c6c4a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc0162f98b8 sp=0xc0162f9888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0078c5110?}, {0x52f0080?, 0xc004737d10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0162f98f8 sp=0xc0162f98b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0078c5110?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc0162f9930 sp=0xc0162f98f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0078c5110}, {0x203003?, 0x40?}, 0xc00dcbb9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc0162f99a8 sp=0xc0162f9930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0078c5110?}, {0x52f0080?, 0xc004737d10?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc0162f9a20 sp=0xc0162f99a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0078c5110?}, {0x52f0080?, 0xc004737d10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc0162f9a60 sp=0xc0162f9a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc0078c5110}, {0x52f0080, 0xc004737d10}, 0xc00f69f020, 0xc001e5b2f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc0162f9ab0 sp=0xc0162f9a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc0078c5110}, 0xc011e16240, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc0162f9b08 sp=0xc0162f9ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc006844fc0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc0162f9e48 sp=0xc0162f9b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0035c51e0}, 0xc006844fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc0162f9f68 sp=0xc0162f9e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc0162f9fe0 sp=0xc0162f9f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0162f9fe8 sp=0xc0162f9fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 117880 [select]: runtime.gopark(0xc010c93f00?, 0x2?, 0x57?, 0x4?, 0xc010c93ed4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc010c93d60 sp=0xc010c93d40 pc=0x4907d6 runtime.selectgo(0xc010c93f00, 0xc010c93ed0, 0x7b55a5?, 0x0, 0xc00fb1c000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc010c93ea0 sp=0xc010c93d60 pc=0x4a071c google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00f9660a0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x115 fp=0xc010c93f30 sp=0xc010c93ea0 pc=0xbd68d5 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00f30db00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 fp=0xc010c93f88 sp=0xc010c93f30 pc=0xbd6fe5 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x6a fp=0xc010c93fe0 sp=0xc010c93f88 pc=0xbe078a runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc010c93fe8 sp=0xc010c93fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x2011 goroutine 488390 [runnable]: runtime.gopark(0xc001836978?, 0x46159f?, 0xc0?, 0xbb?, 0x203001?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001836900 sp=0xc0018368e0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc00b430d50, 0xf1?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc001836968 sp=0xc001836900 pc=0x4a17de sync.runtime_Semacquire(0x11872a5?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc001836998 sp=0xc001836968 pc=0x4be005 sync.(*WaitGroup).Wait(0xc0018369f8?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc0018369c0 sp=0xc001836998 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc00b430d40) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc0018369d8 sp=0xc0018369c0 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00b430d40?, {0x624a8a0?, 0xc00b430d00?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc001836a08 sp=0xc0018369d8 pc=0x118790a github.com/cockroachdb/cockroach/pkg/ccl/backupccl.runBackupProcessor({0x624a948, 0xc00e2edb90}, 0xc0100cae00, 0xc0172d6cc0, 0xc0044a06c0, 0xc00ebc4150) github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backup_processor.go:578 +0x505 fp=0xc001836ee0 sp=0xc001836a08 pc=0x3a85265 github.com/cockroachdb/cockroach/pkg/ccl/backupccl.(*backupDataProcessor).Start.func2({0x624a948?, 0xc00e2edb90?}) github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backup_processor.go:177 +0x58 fp=0xc001836f30 sp=0xc001836ee0 pc=0x3a848d8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001836fe0 sp=0xc001836f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001836fe8 sp=0xc001836fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 345103 [runnable]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x27d?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f174f58 sp=0xc00f174f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x64538fa0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00f174f90 sp=0xc00f174f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f2764538eb8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00f174fb0 sp=0xc00f174f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc015a78400?, 0xc010215000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00f174fd8 sp=0xc00f174fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc015a78400, {0xc010215000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00f175058 sp=0xc00f174fd8 pc=0x535efa net.(*netFD).Read(0xc015a78400, {0xc010215000?, 0x7b5a68?, 0xc015a78400?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00f1750a0 sp=0xc00f175058 pc=0x7a20c9 net.(*conn).Read(0xc01265e128, {0xc010215000?, 0x164290f?, 0xc01265e128?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00f1750e8 sp=0xc00f1750a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00fe5ce40, {0xc010215000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00f175130 sp=0xc00f1750e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00e0ecfa8, {0xc010215000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00f1751d0 sp=0xc00f175130 pc=0x370f896 bufio.(*Reader).fill(0xc00f359890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00f175208 sp=0xc00f1751d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00f359890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00f175228 sp=0xc00f175208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00f359e68?, {0x62295c8, 0xc00f359890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00f175260 sp=0xc00f175228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00f359800, 0xc047456df0, 0xc00f175442, 0xc00f175444, {0x623b718, 0xc004f64b40}, 0xc0059fa000, 0xc00409d004, 0xc00f175440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00f175398 sp=0xc00f175260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00f359800, {0x624a948, 0xc01dc58690}, 0xc047456de0, 0xc0059fa000, 0xc01dc585d0, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00f175768 sp=0xc00f175398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc01dc58690}, {0x628a538, 0xc00fe5ce40}, {{{0xc00a70c009, 0x4}}, 0x1, 0xc01dc58600, 0xc01dc58630, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00f1758a8 sp=0xc00f175768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc01dc58690}, {0x628a538?, 0xc00fe5ce40}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00f175e20 sp=0xc00f1758a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc01dc58510?}, {0x628a538, 0xc00fe5ce40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00f175eb0 sp=0xc00f175e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc01dc58510}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00f175f30 sp=0xc00f175eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f175fe0 sp=0xc00f175f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f175fe8 sp=0xc00f175fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 49079 [select, 7 minutes]: runtime.gopark(0xc0056fbfb0?, 0x2?, 0x60?, 0xc7?, 0xc0056fbf94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0106cbe18 sp=0xc0106cbdf8 pc=0x4907d6 runtime.selectgo(0xc0106cbfb0, 0xc0056fbf90, 0xc00a4ef880?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0106cbf58 sp=0xc0106cbe18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc0106cbfe0 sp=0xc0106cbf58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0106cbfe8 sp=0xc0106cbfe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 120496 [select]: runtime.gopark(0xc001833a08?, 0x2?, 0xe8?, 0xf0?, 0xc0018339ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001833870 sp=0xc001833850 pc=0x4907d6 runtime.selectgo(0xc001833a08, 0xc0018339e8, 0x16e9e6c?, 0x0, 0xc010ff2217?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0018339b0 sp=0xc001833870 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc018838e60, {0xc0183990c0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x90 fp=0xc001833a38 sp=0xc0018339b0 pc=0xbf96b0 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc018838e60, {0xc0183990c0?, 0xc007947200?, 0xc001833a80?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x178 fp=0xc001833a78 sp=0xc001833a38 pc=0xbf9578 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc018348330, {0xc0183990c0?, 0xc001833af8?, 0xbf0787?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc001833ac0 sp=0xc001833a78 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc018348330}, {0xc0183990c0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc001833b08 sp=0xc001833ac0 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc017ef90e0, {0xc0183990c0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc001833b48 sp=0xc001833b08 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc0183990b0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc001833bc8 sp=0xc001833b48 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0x4b588c0?, 0xc017ef90e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc001833c60 sp=0xc001833bc8 pc=0xc19546 google.golang.org/grpc.recv(0xc001833d10?, {0x7f275f301038, 0xc0000af170}, 0xc001833cf8?, {0x0?, 0x0?}, {0x512da60, 0xc008356060}, 0x478065?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc001833cd8 sp=0xc001833c60 pc=0xc19c2e google.golang.org/grpc.(*serverStream).RecvMsg(0xc017d48dd0, {0x512da60?, 0xc008356060}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x178 fp=0xc001833e08 sp=0xc001833cd8 pc=0xc30178 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingServerStream).RecvMsg(0x4618e7?, {0x512da60?, 0xc008356060?}) :1 +0x34 fp=0xc001833e30 sp=0xc001833e08 pc=0x1649cb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc017712330) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:263 +0x4c fp=0xc001833e60 sp=0xc001833e30 pc=0x1d4b66c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:91 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x628a958?, 0xc017712330}, 0xc000c62700, {0x624a948, 0xc018348510}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:289 +0xa6 fp=0xc001833ee0 sp=0xc001833e60 pc=0x1d82d66 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x624a948?, 0xc018348510?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:308 +0x45 fp=0xc001833f30 sp=0xc001833ee0 pc=0x1d82c65 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc001833fe0 sp=0xc001833f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001833fe8 sp=0xc001833fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 450262 [sleep]: runtime.gopark(0x4b85aad2fc0?, 0x400e83?, 0x23?, 0xc8?, 0x801e4bc083?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00150cdf0 sp=0xc00150cdd0 pc=0x4907d6 time.Sleep(0x11fb0faea) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00150ce30 sp=0xc00150cdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00d46b4a0, 0x400e83, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00150ce88 sp=0xc00150ce30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00d46b4a0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00150cec8 sp=0xc00150ce88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00555f500?, 0x26, 0xc001366f98?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00150cfa8 sp=0xc00150cec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00150cfe0 sp=0xc00150cfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00150cfe8 sp=0xc00150cfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498515 [select]: runtime.gopark(0xc012251208?, 0x2?, 0x28?, 0xb7?, 0xc012251204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc012251090 sp=0xc012251070 pc=0x4907d6 runtime.selectgo(0xc012251208, 0xc012251200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0122511d0 sp=0xc012251090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0052ee900) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc012251238 sp=0xc0122511d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc030827d90, {0x51b9a20?, 0xc0533f6f00}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc012251340 sp=0xc012251238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa6?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc012251370 sp=0xc012251340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0052ee5a0, 0xc012251448, 0xc012251418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc0122513c8 sp=0xc012251370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0052ee5a0, {0x51b9a20?, 0xc0533f6f00?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc012251478 sp=0xc0122513c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03ea9eea0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc003bee780}, {0x51b9a20, 0xc0533f6f00}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc0122514e0 sp=0xc012251478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03ea9eea0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc003bee780}, {0x51b9a20, 0xc0533f6f00}, 0x4001e9b2853e6c48?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc0122515d0 sp=0xc0122514e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc03ea9eea0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc003bee780?}, {0x51b9a20?, 0xc0533f6f00?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc012251670 sp=0xc0122515d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043277638, {0x624a948, 0xc03ea9eea0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc0122516f0 sp=0xc012251670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043277638?}}, {0x624a948, 0xc03ea9eea0}, 0xc003bee780, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc0122517c8 sp=0xc0122516f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc03ea9eea0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc012251818 sp=0xc0122517c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00df7b0e0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc012251ab8 sp=0xc012251818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00df7b0e0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51fff78, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc012251bf0 sp=0xc012251ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51fff78, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc012252338 sp=0xc012251bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51fff78, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc012252c38 sp=0xc012252338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51fff78, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc0122534a0 sp=0xc012252c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e51fff78, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc0122538c0 sp=0xc0122534a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc0122539f8 sp=0xc0122538c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc03234bb60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc012253c00 sp=0xc0122539f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc03234bb60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc012253f30 sp=0xc012253c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc012253fe0 sp=0xc012253f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc012253fe8 sp=0xc012253fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 414747 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x299?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00de48f58 sp=0xc00de48f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb79f0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00de48f90 sp=0xc00de48f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb7908, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00de48fb0 sp=0xc00de48f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00d9c9f80?, 0xc00d076000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00de48fd8 sp=0xc00de48fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00d9c9f80, {0xc00d076000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00de49058 sp=0xc00de48fd8 pc=0x535efa net.(*netFD).Read(0xc00d9c9f80, {0xc00d076000?, 0x7b5a68?, 0xc00d9c9f80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00de490a0 sp=0xc00de49058 pc=0x7a20c9 net.(*conn).Read(0xc005168fc8, {0xc00d076000?, 0x164290f?, 0xc005168fc8?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00de490e8 sp=0xc00de490a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00d321380, {0xc00d076000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00de49130 sp=0xc00de490e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00d4482a0, {0xc00d076000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00de491d0 sp=0xc00de49130 pc=0x370f896 bufio.(*Reader).fill(0xc003579490) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00de49208 sp=0xc00de491d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003579490) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00de49228 sp=0xc00de49208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003579a68?, {0x62295c8, 0xc003579490}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00de49260 sp=0xc00de49228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003579400, 0xc00d62ab00, 0xc00de49442, 0xc00de49444, {0x623b718, 0xc005288f30}, 0xc0059fa000, 0xc0070641a4, 0xc00de49440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00de49398 sp=0xc00de49260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003579400, {0x624a948, 0xc0039bbb30}, 0xc00d62aae0, 0xc0059fa000, 0xc0039bba70, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00de49768 sp=0xc00de49398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0039bbb30}, {0x628a538, 0xc00d321380}, {{{0xc0049ac046, 0x4}}, 0x1, 0xc0039bbaa0, 0xc0039bbad0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00de498a8 sp=0xc00de49768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0039bbb30}, {0x628a538?, 0xc00d321380}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00de49e20 sp=0xc00de498a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc0039bb980?}, {0x628a538, 0xc00d321380}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00de49eb0 sp=0xc00de49e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc0039bb980}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00de49f30 sp=0xc00de49eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00de49fe0 sp=0xc00de49f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00de49fe8 sp=0xc00de49fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115783 [select, 6 minutes]: runtime.gopark(0xc00dd1fe58?, 0x2?, 0x3?, 0x30?, 0xc00dd1fdfc?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dd1fc80 sp=0xc00dd1fc60 pc=0x4907d6 runtime.selectgo(0xc00dd1fe58, 0xc00dd1fdf8, 0xc00c7c5280?, 0x0, 0xc00dd1fe58?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dd1fdc0 sp=0xc00dd1fc80 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x624a8a0, 0xc00c7c5280}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:203 +0x15e fp=0xc00dd1ff58 sp=0xc00dd1fdc0 pc=0x19d27fe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00dd1ff78 sp=0xc00dd1ff58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00dd1ffe0 sp=0xc00dd1ff78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dd1ffe8 sp=0xc00dd1ffe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 484648 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc016b4d640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016b4d5d0 sp=0xc016b4d5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00cda7490, 0x4a) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc016b4d618 sp=0xc016b4d5d0 pc=0x4be30c sync.(*Cond).Wait(0xc00312c800?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc016b4d650 sp=0xc016b4d618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00312ce68) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc016b4d708 sp=0xc016b4d650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00a718c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc016b4dcb8 sp=0xc016b4d708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00a718c00, {0x624a8a0, 0xc00cda74c0}, 0x4115a15000000000?, 0xc0085bfdc0?, 0xc0138fc5d0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc016b4dd70 sp=0xc016b4dcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00cda74c0?}, {0xc0059fa000?}, 0x3?, 0xc0014d47c8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc016b4de20 sp=0xc016b4dd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc016b4dfe0 sp=0xc016b4de20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016b4dfe8 sp=0xc016b4dfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 346118 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc010763640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0124d95d0 sp=0xc0124d95b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00a2b1550, 0x281) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc0124d9618 sp=0xc0124d95d0 pc=0x4be30c sync.(*Cond).Wait(0xc00370c000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc0124d9650 sp=0xc0124d9618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00370c668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc0124d9708 sp=0xc0124d9650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0057d6c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc0124d9cb8 sp=0xc0124d9708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0057d6c00, {0x624a8a0, 0xc00a2b1580}, 0x4118331c00000000?, 0xc00a293dc0?, 0xc0153330e0?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc0124d9d70 sp=0xc0124d9cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00a2b1580?}, {0xc0059fa000?}, 0x3?, 0xc00603d638?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc0124d9e20 sp=0xc0124d9d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc0124d9fe0 sp=0xc0124d9e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0124d9fe8 sp=0xc0124d9fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 115741 [semacquire, 6 minutes]: runtime.gopark(0x18?, 0xc003440400?, 0xe0?, 0x56?, 0x4c022e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0044dbaa8 sp=0xc0044dba88 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc00c7c5190, 0x50?, 0x1, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc0044dbb10 sp=0xc0044dbaa8 pc=0x4a17de sync.runtime_Semacquire(0xc00c7c5180?) GOROOT/src/runtime/sema.go:62 +0x25 fp=0xc0044dbb40 sp=0xc0044dbb10 pc=0x4be005 sync.(*WaitGroup).Wait(0x624a8a0?) GOROOT/src/sync/waitgroup.go:139 +0x52 fp=0xc0044dbb68 sp=0xc0044dbb40 pc=0x4ce332 golang.org/x/sync/errgroup.(*Group).Wait(0xc00c7c5180) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:52 +0x27 fp=0xc0044dbb80 sp=0xc0044dbb68 pc=0x11871a7 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00c7c5180?, {0x624a8a0?, 0xc00c7c5140?}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a fp=0xc0044dbbb0 sp=0xc0044dbb80 pc=0x118790a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002aac500, {0x624a8a0, 0xc00c7c4640}, {0xc00c7c5100, 0x1, 0xc003f8e900?}, 0x1, 0xc003f8e8a0, {0xc008c87830, 0x1, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:229 +0x666 fp=0xc0044dbd38 sp=0xc0044dbbb0 pc=0x19d2246 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00bbcced0?, {0x624a8a0, 0xc00c7c4640}, {0xc002e4a900, 0x1, 0x140?}, {0xc00e35a480?, 0x9c13c80?, 0x0?}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:147 +0x108 fp=0xc0044dbe58 sp=0xc0044dbd38 pc=0x19d1988 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc00bbccf30?, {0x624a8a0?, 0xc00c7c4640?}, {0xc002e4a900?, 0x1a01418?, 0x106258000?}, {0xc003f56a10?, 0xbbccf50?, 0xc0?}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x71 fp=0xc0044dbed0 sp=0xc0044dbe58 pc=0x19fd411 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x624a8a0?, 0xc00c7c4640?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:305 +0xc7 fp=0xc0044dbf58 sp=0xc0044dbed0 pc=0x1a01507 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc0044dbf78 sp=0xc0044dbf58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc0044dbfe0 sp=0xc0044dbf78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0044dbfe8 sp=0xc0044dbfe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 138353 [sleep]: runtime.gopark(0x4b85d0d8739?, 0x400ea0?, 0xb6?, 0x4d?, 0x8022064614?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005a4cdf0 sp=0xc005a4cdd0 pc=0x4907d6 time.Sleep(0x1210b26aa) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc005a4ce30 sp=0xc005a4cdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc015e87e90, 0x400ea0, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc005a4ce88 sp=0xc005a4ce30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc015e87e90, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc005a4cec8 sp=0xc005a4ce88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00da26f80?, {0xc0029dca80?, 0xae, 0xc00da26f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc005a4cfa8 sp=0xc005a4cec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc005a4cfe0 sp=0xc005a4cfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005a4cfe8 sp=0xc005a4cfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 495993 [sleep]: runtime.gopark(0x4b87a5fb929?, 0x2005614?, 0x65?, 0xb2?, 0x803ef9aabd?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004a0ddf0 sp=0xc004a0ddd0 pc=0x4907d6 time.Sleep(0x120ac5cc7) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004a0de30 sp=0xc004a0ddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc03f2ceba0, 0x2005614, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004a0de88 sp=0xc004a0de30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc03f2ceba0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004a0dec8 sp=0xc004a0de88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc000059f80?, {0xc0074caa00?, 0x17, 0xc000059f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004a0dfa8 sp=0xc004a0dec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004a0dfe0 sp=0xc004a0dfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004a0dfe8 sp=0xc004a0dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498447 [runnable]: runtime.gopark(0xae1b6688ce0?, 0xc00e02afc0?, 0x0?, 0x0?, 0xffff?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc013484918 sp=0xc0134848f8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x20?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc013484980 sp=0xc013484918 pc=0x4a17de sync.runtime_SemacquireMutex(0x115345e?, 0xd2?, 0xc0134849d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc0134849b0 sp=0xc013484980 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc013484a08 sp=0xc0134849b0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0?, 0x1801f9e?, 0x7f270c2d5aa0?, 0x3cc5?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc013484a50 sp=0xc013484a08 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00858db80, {0x8?, 0x11?}, 0x0, 0x0, 0xc013d2dc10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc013484b50 sp=0xc013484a50 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2727 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc00dca1500, {0xc03c437cc0, 0x20, 0x50}, {0xc0159a6320, 0x20, 0x20}, 0x60?, 0x61?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:847 +0xf8 fp=0xc013484bb8 sp=0xc013484b50 pc=0x10c5078 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc013d2ed40?, {0xc03c437cc0?, 0x1?, 0x20?}, {0xc0159a6320?, 0x10?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc013484c08 sp=0xc013484bb8 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc013d2ed40, {0xc03c437cc0, 0x20, 0x50}, {0xc0159a6320, 0x20, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc013484c58 sp=0xc013484c08 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc013d2de20, {0xc0159a6320?, 0xc00f039e30?, 0x24?}, 0x1f?, 0x1f?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc013484d88 sp=0xc013484c58 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc013d2de20, {0xc03c437cc0?, 0x0?, 0x20?}, {0xc0159a6320?, 0x0?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc013484dc8 sp=0xc013484d88 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc013d2dc88, {0xc03c437cc0?, 0x50?, 0xc0159a6320?}, {0xc0159a6320, 0x20, 0x20}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc013484e20 sp=0xc013484dc8 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc013d2d900, {0xc0159a6320, 0x20, 0x20}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc013484eb8 sp=0xc013484e20 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc007137828, {{0xc00348d4d0, 0x1f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc013484f08 sp=0xc013484eb8 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x62b4fe0, 0xc007137828}, {{0xc00348d4d0, 0x1f, 0x30}, {0x0, 0x0, 0x0}}, 0xc03b446240) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1129 +0xc3 fp=0xc013485358 sp=0xc013484f08 pc=0x1817843 github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x624a948, 0xc020408510}, {0x62b8438?, 0xc007137800?}, {0x62b4fe0, 0xc007137828}, 0xc00c6846e0, {0xc00348d4d0, 0x1f, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1650 +0x165 fp=0xc013486068 sp=0xc013485358 pc=0x181aa25 github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x624a948, 0xc020408510}, {0x62b8438, 0xc007137800}, {0x62b4fe0, 0xc007137828}, 0x129?, {0xc00348d4d0, 0x1f, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1414 +0x1cd fp=0xc013486170 sp=0xc013486068 pc=0x18197cd github.com/cockroachdb/cockroach/pkg/storage.MVCCPut({0x624a948, 0xc020408510}, {0x7f275f2b7158?, 0xc007137800}, 0xc00c6846e0, {0xc00348d4d0, 0x1f, 0x30}, {0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1333 +0x2a5 fp=0xc013486280 sp=0xc013486170 pc=0x1819005 github.com/cockroachdb/cockroach/pkg/storage.MVCCPutProto({0x624a948, 0xc020408510}, {0x7f275f2b7158, 0xc007137800}, 0xc00e8208ad?, {0xc00348d4d0, 0x1f, 0x30}, {0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:841 +0x1a5 fp=0xc013486358 sp=0xc013486280 pc=0x1815d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.updateStagingTxn({0x624a948, 0xc020408510}, {0x7f275f2b7158, 0xc007137800}, 0x0?, {0xc00348d4d0, 0x1f, 0x30}, 0x0?, 0xc006bbe480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:639 +0x33d fp=0xc013486570 sp=0xc013486358 pc=0x1cea5bd github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.EndTxn({_, _}, {_, _}, {{0x62cc4b0, 0xc01106e520}, {{0x17600a764d68fcff, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:360 +0xe12 fp=0xc013487bf0 sp=0xc013486570 pc=0x1ce8532 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a764d68fcff, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc013488480 sp=0xc013487bf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc017226690, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc01348a2f0 sp=0xc013488480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0065985a0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc01348a940 sp=0xc01348a2f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc0065985a0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc01348ae18 sp=0xc01348a940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a766b3661ff, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc01348bc98 sp=0xc01348ae18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000580000?, {0x624a948, 0xc020408510}, {0xc010cb8538, 0x8}, 0xc017226690, 0x8?, 0x0?, {{0x17600a766b3661ff, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc01348c918 sp=0xc01348bc98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc020408510}, {0xc010cb8538, 0x8}, 0xc017226690, 0x400000004?, 0xc01348d968, {{0x17600a766b3661ff, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc01348c9d0 sp=0xc01348c918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc006f10000, {0x624a948, 0xc020408510}, 0xc017226690, 0xc0065985a0, 0xc01348d968, {{0x17600a766b3661ff, 0x0, 0x0}, {0x17600a76556c3273, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc01348d398 sp=0xc01348c9d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc020408510}, 0xc017226690, 0xc0065985a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc01348db08 sp=0xc01348d398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc020408510}, 0xc017226690, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc01348de28 sp=0xc01348db08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a764d68fcff, 0x0, 0x0}, 0x0, {0x17600a76e3aba611, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc01348e1f0 sp=0xc01348de28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a764d68fcff, 0x0, 0x0}, 0x0, {0x17600a76e3aba611, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc01348eb78 sp=0xc01348e1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x6?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3aba611, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc01348edc0 sp=0xc01348eb78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc020408480}, {0x6231bd8?}, 0xc0172265a0) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc01348f430 sp=0xc01348edc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc020408390}, 0xc0172265a0) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc01348f4d8 sp=0xc01348f430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc020408390}, {0x52f0080?, 0xc0172265a0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc01348f518 sp=0xc01348f4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc020408390}, {0x52f0080, 0xc0172265a0}, 0xc010daa040, 0xc01052b7d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc01348f848 sp=0xc01348f518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc020408390?}, {0x52f0080?, 0xc0172265a0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc01348f888 sp=0xc01348f848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc020408390}, {0x52f0080, 0xc0172265a0}, 0xc01348f920?, 0xc00511c000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc01348f8b8 sp=0xc01348f888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc020408390?}, {0x52f0080?, 0xc0172265a0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc01348f8f8 sp=0xc01348f8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc020408390?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc01348f930 sp=0xc01348f8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc020408390}, {0x7f278cfaaf18?, 0x40?}, 0xc01348f9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc01348f9a8 sp=0xc01348f930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc020408390?}, {0x52f0080?, 0xc0172265a0?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc01348fa20 sp=0xc01348f9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc020408390?}, {0x52f0080?, 0xc0172265a0?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc01348fa60 sp=0xc01348fa20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc020408390}, {0x52f0080, 0xc0172265a0}, 0xc010daa040, 0xc01052b7d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc01348fab0 sp=0xc01348fa60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc020408390}, 0xc0077af3e0, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc01348fb08 sp=0xc01348fab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc0069778c0, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc01348fe48 sp=0xc01348fb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc0069778c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc01348ff68 sp=0xc01348fe48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc01348ffe0 sp=0xc01348ff68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01348ffe8 sp=0xc01348ffe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 115790 [select, 6 minutes]: runtime.gopark(0xc004aa2fb0?, 0x2?, 0x0?, 0xe4?, 0xc004aa2f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004aa2e28 sp=0xc004aa2e08 pc=0x4907d6 runtime.selectgo(0xc004aa2fb0, 0xc004aa2f98, 0x9172428?, 0x0, 0xc0f2e5?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004aa2f68 sp=0xc004aa2e28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc004aa2fe0 sp=0xc004aa2f68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004aa2fe8 sp=0xc004aa2fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 51375 [select, 7 minutes]: runtime.gopark(0xc0079d3508?, 0x2?, 0x0?, 0x4?, 0xc0079d34f4?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01610b368 sp=0xc01610b348 pc=0x4907d6 runtime.selectgo(0xc01610b508, 0xc0079d34f0, 0xc00dc6f320?, 0x0, 0x23?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01610b4a8 sp=0xc01610b368 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc011c5b6b0, {0x624a948, 0xc00dc6f320}, 0xc002a40360, {0x628ab10?, 0xc0138fdf40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1b9 fp=0xc01610b560 sp=0xc01610b4a8 pc=0x1d1a619 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc00742cb40, {0x628ab10, 0xc0138fdf40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x92 fp=0xc01610b5b0 sp=0xc01610b560 pc=0x1d18f12 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x51002e0?, 0xc00742cb40}, {0x627d570?, 0xc00a0d5c80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:344 +0x9f fp=0xc01610b5e8 sp=0xc01610b5b0 pc=0x16c585f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x51002e0, 0xc00742cb40}, {0x627dc78?, 0xc011296410?}, 0xc00e9f6918, 0x5683fc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 fp=0xc01610b920 sp=0xc01610b5e8 pc=0x16481a4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc011296410?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b fp=0xc01610b960 sp=0xc01610b920 pc=0xc21dfb github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x51002e0, 0xc00742cb40}, {0x627dc78, 0xc011296410}, 0x1054f90?, 0xc00f12e400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:272 +0x74 fp=0xc01610b990 sp=0xc01610b960 pc=0x16576d4 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc011296410?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc01610b9d0 sp=0xc01610b990 pc=0xc21e23 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:241 +0x2d fp=0xc01610ba00 sp=0xc01610b9d0 pc=0x165798d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc00dc6f260}, {0xc0079d3b08?, 0x46159f?}, 0xc0079d3ac0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc01610ba78 sp=0xc01610ba00 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x51002e0, 0xc00742cb40}, {0x627dc78?, 0xc011296410?}, 0xc00e9f6918, 0xc00f12e400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:240 +0xef fp=0xc01610bb00 sp=0xc01610ba78 pc=0x165790f google.golang.org/grpc.chainStreamInterceptors.func1.1({0x51002e0?, 0xc00742cb40?}, {0x627dc78?, 0xc011296410?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 fp=0xc01610bb40 sp=0xc01610bb00 pc=0xc21e23 google.golang.org/grpc.chainStreamInterceptors.func1({0x51002e0, 0xc00742cb40}, {0x627dc78, 0xc011296410}, 0xc00e9f6918, 0x5683fc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b fp=0xc01610bb90 sp=0xc01610bb40 pc=0xc21d4b google.golang.org/grpc.(*Server).processStreamingRPC(0xc00296b6c0, {0x6294e00, 0xc0074271e0}, 0xc007735320, 0xc000acf290, 0x8fc0b80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xf3b fp=0xc01610be48 sp=0xc01610bb90 pc=0xc22dbb google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0074271e0}, 0xc007735320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9ea fp=0xc01610bf68 sp=0xc01610be48 pc=0xc245ea google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc01610bfe0 sp=0xc01610bf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01610bfe8 sp=0xc01610bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 492347 [sleep]: runtime.gopark(0x4b83ede7d21?, 0x200550a?, 0x52?, 0x35?, 0x7ff67b63a5?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004c98df0 sp=0xc004c98dd0 pc=0x4907d6 time.Sleep(0x113af5188) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004c98e30 sp=0xc004c98df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc001cbc900, 0x200550a, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004c98e88 sp=0xc004c98e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc001cbc900, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004c98ec8 sp=0xc004c98e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc006d2d180?, 0x2, 0xc004c98f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004c98fa8 sp=0xc004c98ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004c98fe0 sp=0xc004c98fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004c98fe8 sp=0xc004c98fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 48665 [runnable]: runtime.gopark(0xc004c97d80?, 0xb?, 0x0?, 0x0?, 0x26f?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004c97a80 sp=0xc004c97a60 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x3e?, 0x0?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc004c97ab8 sp=0xc004c97a80 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f3004a0, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc004c97ad8 sp=0xc004c97ab8 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc011d1c600?, 0xc00415c000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc004c97b00 sp=0xc004c97ad8 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011d1c600, {0xc00415c000, 0x8000, 0x8000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc004c97b80 sp=0xc004c97b00 pc=0x535efa net.(*netFD).Read(0xc011d1c600, {0xc00415c000?, 0x0?, 0x4cd0fe?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc004c97bc8 sp=0xc004c97b80 pc=0x7a20c9 net.(*conn).Read(0xc00c354708, {0xc00415c000?, 0x54ba0a?, 0x0?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc004c97c10 sp=0xc004c97bc8 pc=0x7b53c5 bufio.(*Reader).Read(0xc0155fd800, {0xc005004740, 0x9, 0x0?}) GOROOT/src/bufio/bufio.go:237 +0x1bb fp=0xc004c97c48 sp=0xc004c97c10 pc=0x5aeafb io.ReadAtLeast({0x62052c0, 0xc0155fd800}, {0xc005004740, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:332 +0x9a fp=0xc004c97c90 sp=0xc004c97c48 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc005004740?, 0x9?, 0xbe6d72?}, {0x62052c0?, 0xc0155fd800?}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e fp=0xc004c97ce0 sp=0xc004c97c90 pc=0xba1d8e golang.org/x/net/http2.(*Framer).ReadFrame(0xc005004700) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 fp=0xc004c97d90 sp=0xc004c97ce0 pc=0xba25f5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0041a6d20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x4be fp=0xc004c97fc8 sp=0xc004c97d90 pc=0xbea79e google.golang.org/grpc/internal/transport.newHTTP2Client.func7() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x26 fp=0xc004c97fe0 sp=0xc004c97fc8 pc=0xbe08a6 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004c97fe8 sp=0xc004c97fe0 pc=0x4c2401 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x19bf goroutine 415066 [sleep]: runtime.gopark(0x4b851544ae0?, 0x2008f1?, 0xdc?, 0xac?, 0x801191a530?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004a12df0 sp=0xc004a12dd0 pc=0x4907d6 time.Sleep(0x11c4fc2ed) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004a12e30 sp=0xc004a12df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00b77d290, 0x2008f1, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004a12e88 sp=0xc004a12e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00b77d290, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004a12ec8 sp=0xc004a12e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc004bde000?, 0x2b, 0xc00c246f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004a12fa8 sp=0xc004a12ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004a12fe0 sp=0xc004a12fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004a12fe8 sp=0xc004a12fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 390346 [sleep]: runtime.gopark(0x4b8ab83b392?, 0x2040d?, 0x5d?, 0x23?, 0x80881ce5af?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc000059df0 sp=0xc000059dd0 pc=0x4907d6 time.Sleep(0x138ab9b8a) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc000059e30 sp=0xc000059df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00f6c79e0, 0x2040d, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc000059e88 sp=0xc000059e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00f6c79e0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc000059ec8 sp=0xc000059e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc00eb45500?, 0x32, 0xc001879f70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc000059fa8 sp=0xc000059ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc000059fe0 sp=0xc000059fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000059fe8 sp=0xc000059fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 429079 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x116?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bfc4f58 sp=0xc00bfc4f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7090?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00bfc4f90 sp=0xc00bfc4f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb6fa8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00bfc4fb0 sp=0xc00bfc4f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc011696e80?, 0xc0056ca000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00bfc4fd8 sp=0xc00bfc4fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011696e80, {0xc0056ca000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00bfc5058 sp=0xc00bfc4fd8 pc=0x535efa net.(*netFD).Read(0xc011696e80, {0xc0056ca000?, 0x7b5a68?, 0xc011696e80?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00bfc50a0 sp=0xc00bfc5058 pc=0x7a20c9 net.(*conn).Read(0xc046763738, {0xc0056ca000?, 0x164290f?, 0xc046763738?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00bfc50e8 sp=0xc00bfc50a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0309b6420, {0xc0056ca000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00bfc5130 sp=0xc00bfc50e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc013f303f0, {0xc0056ca000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00bfc51d0 sp=0xc00bfc5130 pc=0x370f896 bufio.(*Reader).fill(0xc004042890) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00bfc5208 sp=0xc00bfc51d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc004042890) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00bfc5228 sp=0xc00bfc5208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc004042e68?, {0x62295c8, 0xc004042890}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00bfc5260 sp=0xc00bfc5228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc004042800, 0xc011d956a0, 0xc00bfc5442, 0xc00bfc5444, {0x623b718, 0xc0079f1ef0}, 0xc0059fa000, 0xc0124ff0e4, 0xc00bfc5440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00bfc5398 sp=0xc00bfc5260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc004042800, {0x624a948, 0xc043373500}, 0xc011d95690, 0xc0059fa000, 0xc043373440, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00bfc5768 sp=0xc00bfc5398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc043373500}, {0x628a538, 0xc0309b6420}, {{{0xc004229038, 0x4}}, 0x1, 0xc043373470, 0xc0433734a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00bfc58a8 sp=0xc00bfc5768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc043373500}, {0x628a538?, 0xc0309b6420}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00bfc5e20 sp=0xc00bfc58a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc043373380?}, {0x628a538, 0xc0309b6420}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00bfc5eb0 sp=0xc00bfc5e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc043373380}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00bfc5f30 sp=0xc00bfc5eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00bfc5fe0 sp=0xc00bfc5f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bfc5fe8 sp=0xc00bfc5fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 476260 [sleep]: runtime.gopark(0x4b8814a1985?, 0x400eba?, 0xe9?, 0x61?, 0x8044915a4a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004a0edf0 sp=0xc004a0edd0 pc=0x4907d6 time.Sleep(0x11f59aa89) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004a0ee30 sp=0xc004a0edf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc007085da0, 0x400eba, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004a0ee88 sp=0xc004a0ee30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc007085da0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004a0eec8 sp=0xc004a0ee88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc036077900?, 0x9, 0xc00143efb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004a0efa8 sp=0xc004a0eec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004a0efe0 sp=0xc004a0efa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004a0efe8 sp=0xc004a0efe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 120414 [select, 6 minutes]: runtime.gopark(0xc0174c1b70?, 0x2?, 0x0?, 0x0?, 0xc0174c1b6c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0174c19f8 sp=0xc0174c19d8 pc=0x4907d6 runtime.selectgo(0xc0174c1b70, 0xc0174c1b68, 0x0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc0174c1b38 sp=0xc0174c19f8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc010ea17a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc0174c1ba0 sp=0xc0174c1b38 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001de24d0, {0x512dba0?, 0xc01743d0e0}, 0xc0153e0800?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc0174c1ca8 sp=0xc0174c1ba0 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc0174c1cd8 sp=0xc0174c1ca8 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc010ea1320, 0xc0174c1db0, 0xc0174c1d80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc0174c1d30 sp=0xc0174c1cd8 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc010ea1320, {0x512dba0?, 0xc01743d0e0?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc0174c1de0 sp=0xc0174c1d30 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0116e4d00, {0x512dba0?, 0xc01743d0e0?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc0174c1e28 sp=0xc0174c1de0 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0160a9e50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:132 +0x4c fp=0xc0174c1e58 sp=0xc0174c1e28 pc=0x1d4b00c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x628a900, 0xc0160a9e50}, 0xc000c62700, {0x624a948, 0xc01743cd50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:409 +0x5b fp=0xc0174c1ee0 sp=0xc0174c1e58 pc=0x1d843fb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x624a948?, 0xc01743cd50?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:424 +0x45 fp=0xc0174c1f30 sp=0xc0174c1ee0 pc=0x1d84345 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc0174c1fe0 sp=0xc0174c1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0174c1fe8 sp=0xc0174c1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 427403 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x479?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc020ee6f58 sp=0xc020ee6f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5fcb7ae0?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc020ee6f90 sp=0xc020ee6f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275fcb79f8, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc020ee6fb0 sp=0xc020ee6f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc03b90a000?, 0xc002bc1000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc020ee6fd8 sp=0xc020ee6fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc03b90a000, {0xc002bc1000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc020ee7058 sp=0xc020ee6fd8 pc=0x535efa net.(*netFD).Read(0xc03b90a000, {0xc002bc1000?, 0x7b5a68?, 0xc03b90a000?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc020ee70a0 sp=0xc020ee7058 pc=0x7a20c9 net.(*conn).Read(0xc016884668, {0xc002bc1000?, 0x164290f?, 0xc016884668?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc020ee70e8 sp=0xc020ee70a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00f74c180, {0xc002bc1000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc020ee7130 sp=0xc020ee70e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03ba779c8, {0xc002bc1000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc020ee71d0 sp=0xc020ee7130 pc=0x370f896 bufio.(*Reader).fill(0xc003860c90) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc020ee7208 sp=0xc020ee71d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc003860c90) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc020ee7228 sp=0xc020ee7208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc003861268?, {0x62295c8, 0xc003860c90}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc020ee7260 sp=0xc020ee7228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc003860c00, 0xc003142b10, 0xc020ee7442, 0xc020ee7444, {0x623b718, 0xc0079f1b00}, 0xc0059fa000, 0xc0056b3914, 0xc020ee7440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc020ee7398 sp=0xc020ee7260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc003860c00, {0x624a948, 0xc03bf38b10}, 0xc003142ab0, 0xc0059fa000, 0xc03bf38a50, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc020ee7768 sp=0xc020ee7398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc03bf38b10}, {0x628a538, 0xc00f74c180}, {{{0xc002bdd038, 0x4}}, 0x1, 0xc03bf38a80, 0xc03bf38ab0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc020ee78a8 sp=0xc020ee7768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc03bf38b10}, {0x628a538?, 0xc00f74c180}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc020ee7e20 sp=0xc020ee78a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc03bf38990?}, {0x628a538, 0xc00f74c180}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc020ee7eb0 sp=0xc020ee7e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc03bf38990}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc020ee7f30 sp=0xc020ee7eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc020ee7fe0 sp=0xc020ee7f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc020ee7fe8 sp=0xc020ee7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498405 [select]: runtime.gopark(0xc00f3dd208?, 0x2?, 0x30?, 0xa3?, 0xc00f3dd204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f3dd090 sp=0xc00f3dd070 pc=0x4907d6 runtime.selectgo(0xc00f3dd208, 0xc00f3dd200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f3dd1d0 sp=0xc00f3dd090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0065f8fc0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00f3dd238 sp=0xc00f3dd1d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc005a47c30, {0x51b9a20?, 0xc00418e580}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00f3dd340 sp=0xc00f3dd238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xdb?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00f3dd370 sp=0xc00f3dd340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0065f8d80, 0xc00f3dd448, 0xc00f3dd418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00f3dd3c8 sp=0xc00f3dd370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0065f8d80, {0x51b9a20?, 0xc00418e580?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00f3dd478 sp=0xc00f3dd3c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc00d93f9b0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc01068e690}, {0x51b9a20, 0xc00418e580}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00f3dd4e0 sp=0xc00f3dd478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc00d93f9b0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc01068e690}, {0x51b9a20, 0xc00418e580}, 0x5a49354b314f69c9?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00f3dd5d0 sp=0xc00f3dd4e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc00d93f9b0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc01068e690?}, {0x51b9a20?, 0xc00418e580?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00f3dd670 sp=0xc00f3dd5d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0137e39e8, {0x624a948, 0xc00d93f9b0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00f3dd6f0 sp=0xc00f3dd670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0137e39e8?}}, {0x624a948, 0xc00d93f9b0}, 0xc01068e690, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00f3dd7c8 sp=0xc00f3dd6f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xb0fb830?, {0x624a948?, 0xc00d93f9b0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00b0fb928?}) :1 +0x65 fp=0xc00f3dd818 sp=0xc00f3dd7c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc013c07bc0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f3ddab8 sp=0xc00f3dd818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc013c07bc0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0d6a107, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f3ddbf0 sp=0xc00f3ddab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0d6a107, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f3de338 sp=0xc00f3ddbf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0d6a107, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f3dec38 sp=0xc00f3de338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0d6a107, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f3df4a0 sp=0xc00f3dec38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0d6a107, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f3df8c0 sp=0xc00f3df4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00f3df9f8 sp=0xc00f3df8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc00d93f980}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00f3dfc00 sp=0xc00f3df9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc00d93f980}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00f3dff30 sp=0xc00f3dfc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f3dffe0 sp=0xc00f3dff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f3dffe8 sp=0xc00f3dffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498309 [select]: runtime.gopark(0xc00a583c18?, 0x3?, 0xd2?, 0x39?, 0xc00a583b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a5838f0 sp=0xc00a5838d0 pc=0x4907d6 runtime.selectgo(0xc00a583c18, 0xc00a583b34, 0xc005f85dc0?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a583a30 sp=0xc00a5838f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc03f839b30}, {0x6221560, 0xc006e74d80}, {0x46?, {0xc00a583e01?, 0xa583e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc00a583da0 sp=0xc00a583a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc03f839b30?}, 0x164, 0xc006e74d80, 0x13?, 0xc000af6880) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc00a583e70 sp=0xc00a583da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc03f839b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc00a583f30 sp=0xc00a583e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00a583fe0 sp=0xc00a583f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a583fe8 sp=0xc00a583fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498445 [runnable]: runtime.gopark(0xae1b642f580?, 0x0?, 0x0?, 0x0?, 0x0?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00e66d9a0 sp=0xc00e66d980 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee58c, 0x0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00e66da08 sp=0xc00e66d9a0 pc=0x4a17de sync.runtime_SemacquireMutex(0x10a571c?, 0x10?, 0x0?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00e66da38 sp=0xc00e66da08 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee570, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00e66da90 sp=0xc00e66da38 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10a35ed?, 0xc00e66db70?, 0xc00e66db20?, 0x10b86d8?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00e66dad8 sp=0xc00e66da90 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc004e6b340, {0xc00e66dc18?, 0x4ccece?}, 0x0, 0x0, 0xc013d2b710) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00e66dbd8 sp=0xc00e66dad8 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2723 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc00203d800, 0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1, ...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:378 +0x8f fp=0xc00e66dc28 sp=0xc00e66dbd8 pc=0x10c37af github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc004e6b340, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0?, 0x0?, 0xc013d2b710) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2634 +0x196 fp=0xc00e66dcb0 sp=0xc00e66dc28 pc=0x10ca3d6 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc002941970, 0x3a5d564?, 0xc013d2c640, {0x0?, 0xc013d2b710?, {0x0?, 0x0?}}, 0xc000932208) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:419 +0x3f0 fp=0xc00e66dde0 sp=0xc00e66dcb0 pc=0x1166a30 github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x7f2581089698?, 0x0?, 0xc006ec97e0?, {0x0?, 0xc013d2b710?, {0x0?, 0x0?}}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:138 +0x5a fp=0xc00e66de30 sp=0xc00e66dde0 pc=0x116577a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters-fm(0xc013d2c5f0?, 0xc005abd080?, {0x0?, 0xc013d2b710?, {0x0?, 0x0?}}) :1 +0x5b fp=0xc00e66de78 sp=0xc00e66de30 pc=0x117c93b github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc013d2c5f0, 0xc005abd080, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:581 +0x2b9 fp=0xc00e66ded8 sp=0xc00e66de78 pc=0x114b519 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc013d2c5f0, {0xc006ec97e0, 0xc, 0x20}, {0xc0040c6ca0, 0xc, 0xc}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:665 +0xd7 fp=0xc00e66df28 sp=0xc00e66ded8 pc=0x114ba77 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc013d2b920, {0xc0040c6ca0?, 0x49e9380?, 0x1?}, 0xc00e66e120?, 0xed?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00e66e058 sp=0xc00e66df28 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc013d2b920, {0xc006ec97e0?, 0xc?, 0xc007071480?}, {0xc0040c6ca0?, 0xc007071480?, 0xb?}, 0x1d?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00e66e098 sp=0xc00e66e058 pc=0x115285d github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc013d2b400, {0xc0040c6ca0, 0xc, 0xc}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00e66e130 sp=0xc00e66e098 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc006ab82d8, {{0xc007071480, 0xb, 0x1d}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:373 +0x147 fp=0xc00e66e188 sp=0xc00e66e130 pc=0x1863127 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc005384500, {{0xc0038eac68, 0x5, 0x8}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:520 +0x4d5 fp=0xc00e66e248 sp=0xc00e66e188 pc=0x180d135 github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x62b4f18, 0xc005384500}, {{0xc0038eac68, 0x5, 0x8}, {0x0, 0x0, 0x0}}, 0xc0120179e0) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1129 +0xc3 fp=0xc00e66e698 sp=0xc00e66e248 pc=0x1817843 github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x624a948, 0xc015350c00}, {0x62b8438?, 0xc006ab8000?}, {0x62b4f18, 0xc005384500}, 0xc00c684320, {0xc0038eac68, 0x5, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1650 +0x165 fp=0xc00e66f3a8 sp=0xc00e66e698 pc=0x181aa25 github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x624a948, 0xc015350c00}, {0x62b8438, 0xc006ab8000}, {0x62b4f18, 0xc005384500}, 0x3?, {0xc0038eac68, 0x5, 0x8}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1414 +0x1cd fp=0xc00e66f4b0 sp=0xc00e66f3a8 pc=0x18197cd github.com/cockroachdb/cockroach/pkg/storage.MVCCPut({0x624a948, 0xc015350c00}, {0x7f275f2b7158?, 0xc006ab8000}, 0xc00c684320, {0xc0038eac68, 0x5, 0x8}, {0x17600a76dfa9ffd9, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1333 +0x2a5 fp=0xc00e66f5c0 sp=0xc00e66f4b0 pc=0x1819005 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Put({_, _}, {_, _}, {{0x62cc4b0, 0xc010d38140}, {{0x17600a76dfa9ffd9, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_put.go:59 +0x311 fp=0xc00e66fbf0 sp=0xc00e66f5c0 pc=0x1cfa251 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, _, {{0x17600a76dfa9ffd9, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:520 +0x329 fp=0xc00e670480 sp=0xc00e66fbf0 pc=0x1dc62a9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00dcec960, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:287 +0xd05 fp=0xc00e6722f0 sp=0xc00e670480 pc=0x1dc4d45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, 0xc015dfe500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:668 +0x205 fp=0xc00e672940 sp=0xc00e6722f0 pc=0x1e18085 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc015dfe500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:628 +0x327 fp=0xc00e672e18 sp=0xc00e672940 pc=0x1e17c67 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, _, _, {{0x17600a76fd7764d9, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:438 +0x7f0 fp=0xc00e673c98 sp=0xc00e672e18 pc=0x1e16550 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc000580000?, {0x624a948, 0xc015350c00}, {0xc0038ead50, 0x8}, 0xc00dcec960, 0x8?, 0x0?, {{0x17600a76fd7764d9, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:716 +0x1df fp=0xc00e674918 sp=0xc00e673c98 pc=0x1dd209f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x1b?, {0x624a948?, 0xc015350c00}, {0xc0038ead50, 0x8}, 0xc00dcec960, 0x400000004?, 0xc00e675968, {{0x17600a76fd7764d9, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:804 +0xb2 fp=0xc00e6749d0 sp=0xc00e674918 pc=0x1dd2cd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc0072d4000, {0x624a948, 0xc015350c00}, 0xc00dcec960, 0xc015dfe500, 0xc00e675968, {{0x17600a76fd7764d9, 0x0, 0x0}, {0x17600a76e06082bf, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:119 +0x18f fp=0xc00e675398 sp=0xc00e6749d0 pc=0x1dda44f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0072d4000, {0x624a948, 0xc015350c00}, 0xc00dcec960, 0xc015dfe500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:174 +0x7aa fp=0xc00e675b08 sp=0xc00e675398 pc=0x1e1470a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0072d4000, {0x624a948, 0xc015350c00}, 0xc00dcec960, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00e675e28 sp=0xc00e675b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0072d4000, {_, _}, {{{0x17600a76dfa9ffd9, 0x0, 0x0}, 0x0, {0x17600a76e3711ad8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00e6761f0 sp=0xc00e675e28 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76dfa9ffd9, 0x0, 0x0}, 0x0, {0x17600a76e3711ad8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00e676b78 sp=0xc00e6761f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0x103?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3711ad8, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00e676dc0 sp=0xc00e676b78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc015350b40}, {0x9c5fde0?}, 0xc00dcec870) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00e677430 sp=0xc00e676dc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc015350990}, 0xc00dcec870) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00e6774d8 sp=0xc00e677430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc015350990}, {0x52f0080?, 0xc00dcec870}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00e677518 sp=0xc00e6774d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc015350990}, {0x52f0080, 0xc00dcec870}, 0xc010c71760, 0xc013dcde90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00e677848 sp=0xc00e677518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc015350990?}, {0x52f0080?, 0xc00dcec870?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00e677888 sp=0xc00e677848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc015350990}, {0x52f0080, 0xc00dcec870}, 0xc00e677920?, 0xc00dad2240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00e6778b8 sp=0xc00e677888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc015350990?}, {0x52f0080?, 0xc00dcec870?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e6778f8 sp=0xc00e6778b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc015350990?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00e677930 sp=0xc00e6778f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc015350990}, {0x203004?, 0x40?}, 0xc00e6779e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00e6779a8 sp=0xc00e677930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc015350990?}, {0x52f0080?, 0xc00dcec870?}, 0x5003b00?, 0x7f278cfaaf01?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00e677a20 sp=0xc00e6779a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc015350990?}, {0x52f0080?, 0xc00dcec870?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00e677a60 sp=0xc00e677a20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc015350990}, {0x52f0080, 0xc00dcec870}, 0xc010c71760, 0xc013dcde90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00e677ab0 sp=0xc00e677a60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc015350990}, 0xc00d1dff20, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00e677b08 sp=0xc00e677ab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc047d0d200, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00e677e48 sp=0xc00e677b08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc047d0d200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00e677f68 sp=0xc00e677e48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00e677fe0 sp=0xc00e677f68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00e677fe8 sp=0xc00e677fe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 491248 [sleep]: runtime.gopark(0x4b8e776da17?, 0x2005626?, 0xe6?, 0xf2?, 0x80dac84942?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a631df0 sp=0xc00a631dd0 pc=0x4907d6 time.Sleep(0x14f63d852) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00a631e30 sp=0xc00a631df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc01827c660, 0x2005626, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00a631e88 sp=0xc00a631e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc01827c660, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00a631ec8 sp=0xc00a631e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc0118052c0?, 0x6, 0xc00a2f2340?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00a631fa8 sp=0xc00a631ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00a631fe0 sp=0xc00a631fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a631fe8 sp=0xc00a631fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 118032 [select, 6 minutes]: runtime.gopark(0xc004e50f98?, 0x2?, 0x0?, 0x0?, 0xc004e50f7c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004e50e00 sp=0xc004e50de0 pc=0x4907d6 runtime.selectgo(0xc004e50f98, 0xc004e50f78, 0xc008aa39c0?, 0x0, 0x0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004e50f40 sp=0xc004e50e00 pc=0x4a071c google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0109b3a00) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x73 fp=0xc004e50fc8 sp=0xc004e50f40 pc=0xc088f3 google.golang.org/grpc.newCCBalancerWrapper.func1() google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x26 fp=0xc004e50fe0 sp=0xc004e50fc8 pc=0xc08846 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004e50fe8 sp=0xc004e50fe0 pc=0x4c2401 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22a goroutine 67584 [select, 2 minutes]: runtime.gopark(0xc00e70fea8?, 0x4?, 0x58?, 0x0?, 0xc00e70fe30?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004c9aca0 sp=0xc004c9ac80 pc=0x4907d6 runtime.selectgo(0xc004c9aea8, 0xc00e70fe28, 0x0?, 0x0, 0x62dcd30?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc004c9ade0 sp=0xc004c9aca0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run(0xc00096dbd0, {0x624a948, 0xc004911860}, 0xc002a40360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:104 +0x27b fp=0xc004c9af00 sp=0xc004c9ade0 pc=0x3738c9b github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).StartReconciler.func1({0x624a948?, 0xc004911860?}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:77 +0x34 fp=0xc004c9af30 sp=0xc004c9af00 pc=0x37389d4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc004c9afe0 sp=0xc004c9af30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004c9afe8 sp=0xc004c9afe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115785 [select]: runtime.gopark(0xc00a594d68?, 0x2?, 0xa0?, 0x4c?, 0xc00a594d34?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00a594bb8 sp=0xc00a594b98 pc=0x4907d6 runtime.selectgo(0xc00a594d68, 0xc00a594d30, 0x8?, 0x0, 0xc0125ae000?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00a594cf8 sp=0xc00a594bb8 pc=0x4a071c google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc007322500, {0xc015b2e640, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xa5 fp=0xc00a594d98 sp=0xc00a594cf8 pc=0xbf9805 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc007322500, {0xc015b2e640?, 0xc005939da0?, 0xc00a594de0?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 fp=0xc00a594dd8 sp=0xc00a594d98 pc=0xbf9547 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00e30fd40, {0xc015b2e640?, 0xc00a594e58?, 0xbe1107?}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 fp=0xc00a594e20 sp=0xc00a594dd8 pc=0xbfabb2 io.ReadAtLeast({0x6213e00, 0xc00e30fd40}, {0xc015b2e640, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:332 +0x9a fp=0xc00a594e68 sp=0xc00a594e20 pc=0x52d13a io.ReadFull(...) GOROOT/src/io/io.go:351 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0115ccb40, {0xc015b2e640, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 fp=0xc00a594ea8 sp=0xc00a594e68 pc=0xbfab25 google.golang.org/grpc.(*parser).recvMsg(0xc015b2e630, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 fp=0xc00a594f28 sp=0xc00a594ea8 pc=0xc18c47 google.golang.org/grpc.recvAndDecompress(0xc00f74d028?, 0xc0115ccb40, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6248888, 0x9c5fde0}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 fp=0xc00a594fc0 sp=0xc00a594f28 pc=0xc19546 google.golang.org/grpc.recv(0x468986?, {0x7f275f301038, 0xc0000af170}, 0x7f274e1edfff?, {0x0?, 0x0?}, {0x51d3160, 0xc025aa9260}, 0x7f274e004700?, 0x0, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e fp=0xc00a595038 sp=0xc00a594fc0 pc=0xc19c2e google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ca5b80, {0x51d3160?, 0xc025aa9260}, 0x1?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2ba fp=0xc00a595140 sp=0xc00a595038 pc=0xc2c5ba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00a595170 sp=0xc00a595140 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc0115cc900, 0xc00a595248, 0xc00a595218) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f1 fp=0xc00a5951c8 sp=0xc00a595170 pc=0xc2a5d1 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0115cc900, {0x51d3160?, 0xc025aa9260?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00a595278 sp=0xc00a5951c8 pc=0xc2b61f github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc008a7fee0, {0x51d3160?, 0xc025aa9260?}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:384 +0x37 fp=0xc00a5952c0 sp=0xc00a595278 pc=0x1649ad7 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc014294ac0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9376 +0x4c fp=0xc00a5952f0 sp=0xc00a5952c0 pc=0xdcfb8c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func6() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:649 +0x2e fp=0xc00a595310 sp=0xc00a5952f0 pc=0x19d6c2e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc014351d60, 0xc00a595780) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 fp=0xc00a5953b8 sp=0xc00a595310 pc=0x19d7622 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002aac500, {0x624a8a0?, 0xc00c7c5140?}, {{0xc004c21648, 0x1, 0x8}, {0xc004c21650, 0x1, 0x8}}, {0x17600a17c78406fa, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:648 +0xffa fp=0xc00a5959a0 sp=0xc00a5953b8 pc=0x19d629a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002aac500, {0x624a8a0, 0xc00c7c5140}, 0xc014351ae0, 0x3?, {{0xc004c21648, 0x1, 0x8}, {0xc004c21650, 0x1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:423 +0x8fb fp=0xc00a595e80 sp=0xc00a5959a0 pc=0x19d467b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x624a8a0?, 0xc00c7c5140?}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:207 +0x117 fp=0xc00a595f58 sp=0xc00a595e80 pc=0x19d2b17 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 fp=0xc00a595f78 sp=0xc00a595f58 pc=0x1187be5 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:74 +0x64 fp=0xc00a595fe0 sp=0xc00a595f78 pc=0x1187344 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00a595fe8 sp=0xc00a595fe0 pc=0x4c2401 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:71 +0xa5 goroutine 428125 [IO wait]: runtime.gopark(0x4d28360?, 0xb?, 0x0?, 0x0?, 0x113?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bf50f58 sp=0xc00bf50f38 pc=0x4907d6 runtime.netpollblock(0x4d6745?, 0x5f300b28?, 0x27?) GOROOT/src/runtime/netpoll.go:526 +0xf7 fp=0xc00bf50f90 sp=0xc00bf50f58 pc=0x4891b7 internal/poll.runtime_pollWait(0x7f275f300a40, 0x72) GOROOT/src/runtime/netpoll.go:305 +0x89 fp=0xc00bf50fb0 sp=0xc00bf50f90 pc=0x4bc6e9 internal/poll.(*pollDesc).wait(0xc00d114980?, 0xc003685000?, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00bf50fd8 sp=0xc00bf50fb0 pc=0x534b92 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00d114980, {0xc003685000, 0x1000, 0x1000}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a fp=0xc00bf51058 sp=0xc00bf50fd8 pc=0x535efa net.(*netFD).Read(0xc00d114980, {0xc003685000?, 0x7b5a68?, 0xc00d114980?}) GOROOT/src/net/fd_posix.go:55 +0x29 fp=0xc00bf510a0 sp=0xc00bf51058 pc=0x7a20c9 net.(*conn).Read(0xc0137e3330, {0xc003685000?, 0x164290f?, 0xc0137e3330?}) GOROOT/src/net/net.go:183 +0x45 fp=0xc00bf510e8 sp=0xc00bf510a0 pc=0x7b53c5 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc010f7db00, {0xc003685000, 0x1000, 0x1000}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 fp=0xc00bf51130 sp=0xc00bf510e8 pc=0x1641f96 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00c9f8258, {0xc003685000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1949 +0x156 fp=0xc00bf511d0 sp=0xc00bf51130 pc=0x370f896 bufio.(*Reader).fill(0xc00d3b1090) GOROOT/src/bufio/bufio.go:106 +0xff fp=0xc00bf51208 sp=0xc00bf511d0 pc=0x5ae51f bufio.(*Reader).ReadByte(0xc00d3b1090) GOROOT/src/bufio/bufio.go:265 +0x2c fp=0xc00bf51228 sp=0xc00bf51208 pc=0x5aecec github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00d3b1668?, {0x62295c8, 0xc00d3b1090}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:206 +0x2c fp=0xc00bf51260 sp=0xc00bf51228 pc=0x270ca4c github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00d3b1000, 0xc01847c0d0, 0xc00bf51442, 0xc00bf51444, {0x623b718, 0xc011db9dd0}, 0xc0059fa000, 0xc00f942fb4, 0xc00bf51440) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:424 +0xa5 fp=0xc00bf51398 sp=0xc00bf51260 pc=0x3707185 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00d3b1000, {0x624a948, 0xc0048f6120}, 0xc01847c0b0, 0xc0059fa000, 0xc0048f6000, {0x1, 0x2, {0x4, {0x532ee51, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:564 +0xc7f fp=0xc00bf51768 sp=0xc00bf51398 pc=0x3706c5f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc005765380, {0x624a948, 0xc0048f6120}, {0x628a538, 0xc010f7db00}, {{{0xc001bbe01f, 0x4}}, 0x1, 0xc0048f6030, 0xc0048f6060, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:174 +0x29e fp=0xc00bf518a8 sp=0xc00bf51768 pc=0x37051fe github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc005765380, {0x624a948, 0xc0048f6120}, {0x628a538?, 0xc010f7db00}, 0x90?) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:913 +0xd29 fp=0xc00bf51e20 sp=0xc00bf518a8 pc=0x3715be9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x624a948?, 0xc030805f80?}, {0x628a538, 0xc010f7db00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1474 +0xc8 fp=0xc00bf51eb0 sp=0xc00bf51e20 pc=0x39e1cc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x624a948, 0xc030805f80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0xd7 fp=0xc00bf51f30 sp=0xc00bf51eb0 pc=0x1643997 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00bf51fe0 sp=0xc00bf51f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bf51fe8 sp=0xc00bf51fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 115797 [select, 6 minutes]: runtime.gopark(0xc00bbccfb0?, 0x2?, 0xfa?, 0x6?, 0xc00bbccf9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00bbcce28 sp=0xc00bbcce08 pc=0x4907d6 runtime.selectgo(0xc00bbccfb0, 0xc00bbccf98, 0xc002a40360?, 0x0, 0x10000c00337f680?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00bbccf68 sp=0xc00bbcce28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc00bbccfe0 sp=0xc00bbccf68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00bbccfe8 sp=0xc00bbccfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 498396 [select]: runtime.gopark(0xc00f5971f0?, 0x2?, 0x0?, 0xc3?, 0xc00f5971ec?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f597078 sp=0xc00f597058 pc=0x4907d6 runtime.selectgo(0xc00f5971f0, 0xc00f5971e8, 0x10?, 0x0, 0x101?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f5971b8 sp=0xc00f597078 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc017ca7d40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00f597220 sp=0xc00f5971b8 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc003874630, {0x51b9a20?, 0xc0178af100}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00f597328 sp=0xc00f597220 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xf9?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00f597358 sp=0xc00f597328 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc017ca79e0, 0xc00f597430, 0xc00f597400) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00f5973b0 sp=0xc00f597358 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc017ca79e0, {0x51b9a20?, 0xc0178af100?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00f597460 sp=0xc00f5973b0 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc03b5dc900?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc0134f0960}, {0x51b9a20, 0xc0178af100}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00f5974c8 sp=0xc00f597460 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc03b5dc900}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0134f0960}, {0x51b9a20, 0xc0178af100}, 0x5d2f0d2045333976?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00f5975b8 sp=0xc00f5974c8 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x1bc79a43c8b5029e?, {0x624a948?, 0xc03b5dc900?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc0134f0960?}, {0x51b9a20?, 0xc0178af100?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00f597658 sp=0xc00f5975b8 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0123be918, {0x624a948, 0xc03b5dc900}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00f5976d8 sp=0xc00f597658 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc0123be918?}}, {0x624a948, 0xc03b5dc900}, 0xc0134f0960, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00f5977b0 sp=0xc00f5976d8 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xd1ad818?, {0x624a948?, 0xc03b5dc900?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00d1ad910?}) :1 +0x65 fp=0xc00f597800 sp=0xc00f5977b0 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc015325920, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f597aa0 sp=0xc00f597800 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc015325920, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b9af13, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f597bd8 sp=0xc00f597aa0 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b9af13, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f598320 sp=0xc00f597bd8 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b9af13, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f598c20 sp=0xc00f598320 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b9af13, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f599488 sp=0xc00f598c20 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e0b9af13, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f5998a8 sp=0xc00f599488 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00ba62bb8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x2a8 fp=0xc00f599b00 sp=0xc00f5998a8 pc=0x19f2348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00ba62b80, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x142 fp=0xc00f599c28 sp=0xc00f599b00 pc=0x19e9042 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x624a948, 0xc00fb4ebd0}, {0x7f275f30db68, 0xc00ba62b80}, 0xc011e05320, {0xc005d9a840, 0x2, 0x4}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x270 fp=0xc00f599e60 sp=0xc00f599c28 pc=0x19e6430 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x624a948, 0xc00fb4ebd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 fp=0xc00f599f30 sp=0xc00f599e60 pc=0x19e6054 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f599fe0 sp=0xc00f599f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f599fe8 sp=0xc00f599fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 120616 [select, 6 minutes]: runtime.gopark(0xc004315fb0?, 0x2?, 0x10?, 0xab?, 0xc004315f94?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc005b05e18 sp=0xc005b05df8 pc=0x4907d6 runtime.selectgo(0xc005b05fb0, 0xc004315f90, 0xc00d57ce00?, 0x0, 0xc002a40360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005b05f58 sp=0xc005b05e18 pc=0x4a071c google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x92 fp=0xc005b05fe0 sp=0xc005b05f58 pc=0xc28912 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005b05fe8 sp=0xc005b05fe0 pc=0x4c2401 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb7e goroutine 384704 [sleep]: runtime.gopark(0x4b902ab2be5?, 0x220741?, 0xf2?, 0x6a?, 0x80eb91c152?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc002e07df0 sp=0xc002e07dd0 pc=0x4907d6 time.Sleep(0x144f8fdb9) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc002e07e30 sp=0xc002e07df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc007b68420, 0x220741, {0x4ccaee?, 0x203001?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc002e07e88 sp=0xc002e07e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc007b68420, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc002e07ec8 sp=0xc002e07e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc003e48a80?, 0x2c, 0xc011573fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc002e07fa8 sp=0xc002e07ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc002e07fe0 sp=0xc002e07fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc002e07fe8 sp=0xc002e07fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 471169 [sleep]: runtime.gopark(0x4b883f4d437?, 0x20096d?, 0xce?, 0x2d?, 0x80481f262c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc004a11df0 sp=0xc004a11dd0 pc=0x4907d6 time.Sleep(0x1203cbccc) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc004a11e30 sp=0xc004a11df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00e323f20, 0x20096d, {0x4618e7?, 0x203003?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc004a11e88 sp=0xc004a11e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00e323f20, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc004a11ec8 sp=0xc004a11e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc01687d400?, 0xd, 0xc006a41fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc004a11fa8 sp=0xc004a11ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc004a11fe0 sp=0xc004a11fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc004a11fe8 sp=0xc004a11fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 435561 [sleep]: runtime.gopark(0x4b8d38101c7?, 0xe060c?, 0xa6?, 0x81?, 0x80ca94d7fb?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00115ddf0 sp=0xc00115ddd0 pc=0x4907d6 time.Sleep(0x153264015) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00115de30 sp=0xc00115ddf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00839e990, 0xe060c, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00115de88 sp=0xc00115de30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00839e990, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00115dec8 sp=0xc00115de88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc004210500?, 0x1f, 0xc006e15fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00115dfa8 sp=0xc00115dec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00115dfe0 sp=0xc00115dfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00115dfe8 sp=0xc00115dfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 393623 [sleep]: runtime.gopark(0x4b88c60f03f?, 0x400e66?, 0x6d?, 0x2?, 0x80645cc4bf?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003fabdf0 sp=0xc003fabdd0 pc=0x4907d6 time.Sleep(0x1340e3c9f) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc003fabe30 sp=0xc003fabdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc011f56180, 0x400e66, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc003fabe88 sp=0xc003fabe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc011f56180, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc003fabec8 sp=0xc003fabe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc002d68000?, 0x2a, 0xc0036b7fb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc003fabfa8 sp=0xc003fabec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc003fabfe0 sp=0xc003fabfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003fabfe8 sp=0xc003fabfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498320 [select]: runtime.gopark(0xc005603c18?, 0x3?, 0xd2?, 0x39?, 0xc005603b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0056038f0 sp=0xc0056038d0 pc=0x4907d6 runtime.selectgo(0xc005603c18, 0xc005603b34, 0xc008152500?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc005603a30 sp=0xc0056038f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc0034e9a40}, {0x6221560, 0xc00559a000}, {0x46?, {0xc005603e01?, 0x5603e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc005603da0 sp=0xc005603a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc0034e9a40?}, 0x164, 0xc00559a000, 0x10?, 0xc0120e6bf0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc005603e70 sp=0xc005603da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc0034e9a40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc005603f30 sp=0xc005603e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc005603fe0 sp=0xc005603f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc005603fe8 sp=0xc005603fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 121101 [select, 6 minutes]: runtime.gopark(0xc007ba1fb0?, 0x2?, 0x0?, 0x87?, 0xc007ba1f9c?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc001c6ce28 sp=0xc001c6ce08 pc=0x4907d6 runtime.selectgo(0xc001c6cfb0, 0xc007ba1f98, 0xc002a40360?, 0x0, 0xc00b1da1a0?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc001c6cf68 sp=0xc001c6ce28 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:340 +0x71 fp=0xc001c6cfe0 sp=0xc001c6cf68 pc=0x1649731 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc001c6cfe8 sp=0xc001c6cfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:339 +0x152 goroutine 488454 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc016ae7640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc016ae75d0 sp=0xc016ae75b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00c62c950, 0x4e) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc016ae7618 sp=0xc016ae75d0 pc=0x4be30c sync.(*Cond).Wait(0xc003918000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc016ae7650 sp=0xc016ae7618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc003918668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc016ae7708 sp=0xc016ae7650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00c4eb900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc016ae7cb8 sp=0xc016ae7708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00c4eb900, {0x624a8a0, 0xc00c62c980}, 0x4117ebf800000000?, 0xc006713dc0?, 0xc00ba6c690?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc016ae7d70 sp=0xc016ae7cb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00c62c980?}, {0xc0059fa000?}, 0x3?, 0xc00602fab8?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc016ae7e20 sp=0xc016ae7d70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc016ae7fe0 sp=0xc016ae7e20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc016ae7fe8 sp=0xc016ae7fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 416813 [sleep]: runtime.gopark(0x4b8d8461fe9?, 0x204d13?, 0x96?, 0xbe?, 0x80cbdf14e7?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00885fdf0 sp=0xc00885fdd0 pc=0x4907d6 time.Sleep(0x14fab60da) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00885fe30 sp=0xc00885fdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0137c8450, 0x204d13, {0x2?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00885fe88 sp=0xc00885fe30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0137c8450, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00885fec8 sp=0xc00885fe88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0x0?, {0xc005f99500?, 0x26, 0xc00dd1af70?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00885ffa8 sp=0xc00885fec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00885ffe0 sp=0xc00885ffa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00885ffe8 sp=0xc00885ffe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498255 [select]: runtime.gopark(0xc007cf1c18?, 0x3?, 0xd2?, 0x19?, 0xc007cf1b3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007cf18f0 sp=0xc007cf18d0 pc=0x4907d6 runtime.selectgo(0xc007cf1c18, 0xc007cf1b34, 0xc00df5a9c0?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc007cf1a30 sp=0xc007cf18f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc014d5dda0}, {0x6221560, 0xc0060da900}, {0x46?, {0xc007cf1e01?, 0x7cf1e60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc007cf1da0 sp=0xc007cf1a30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc014d5dda0?}, 0x164, 0xc0060da900, 0x1?, 0xc00096b360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc007cf1e70 sp=0xc007cf1da0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc014d5dda0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc007cf1f30 sp=0xc007cf1e70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc007cf1fe0 sp=0xc007cf1f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007cf1fe8 sp=0xc007cf1fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 344855 [sync.Cond.Wait]: runtime.gopark(0x4c13fe0?, 0x4fdee40?, 0x12?, 0x0?, 0xc01b80b640?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc003a6b5d0 sp=0xc003a6b5b0 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 sync.runtime_notifyListWait(0xc00e620a10, 0x446) GOROOT/src/runtime/sema.go:517 +0x14c fp=0xc003a6b618 sp=0xc003a6b5d0 pc=0x4be30c sync.(*Cond).Wait(0xc00a6da000?) GOROOT/src/sync/cond.go:70 +0x8c fp=0xc003a6b650 sp=0xc003a6b618 pc=0x4cabac github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00a6da668) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:460 +0xbe fp=0xc003a6b708 sp=0xc003a6b650 pc=0x3321a5e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc002dbf300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2074 +0xe5 fp=0xc003a6bcb8 sp=0xc003a6b708 pc=0x32f8b65 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc002dbf300, {0x624a8a0, 0xc00e620a40}, 0x41185d0400000000?, 0xc00b4bfdc0?, 0xc012352e20?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2048 +0x20b fp=0xc003a6bd70 sp=0xc003a6bcb8 pc=0x32f86eb github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0057ff5e0?, {0x624a8a0?, 0xc00e620a40?}, {0xc0059fa000?}, 0x3?, 0xc002e5f728?) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:881 +0xe6 fp=0xc003a6be20 sp=0xc003a6bd70 pc=0x32f3486 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:758 +0x54f fp=0xc003a6bfe0 sp=0xc003a6be20 pc=0x37088af runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc003a6bfe8 sp=0xc003a6bfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:665 +0x22e goroutine 481500 [sleep]: runtime.gopark(0x4b884027568?, 0x1002f10?, 0xc3?, 0x1b?, 0x804f93de11?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00c24cdf0 sp=0xc00c24cdd0 pc=0x4907d6 time.Sleep(0x127a3d3a1) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc00c24ce30 sp=0xc00c24cdf0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc0087ea2d0, 0x1002f10, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc00c24ce88 sp=0xc00c24ce30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc0087ea2d0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc00c24cec8 sp=0xc00c24ce88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc003fcd380?, {0xc007f40a00?, 0xf, 0x0?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc00c24cfa8 sp=0xc00c24cec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc00c24cfe0 sp=0xc00c24cfa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00c24cfe8 sp=0xc00c24cfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 422669 [sleep]: runtime.gopark(0x4b8c24a9516?, 0x2008fa?, 0x58?, 0x1e?, 0x80ac06aaa7?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc0050e5df0 sp=0xc0050e5dd0 pc=0x4907d6 time.Sleep(0x145ce804b) GOROOT/src/runtime/time.go:195 +0x135 fp=0xc0050e5e30 sp=0xc0050e5df0 pc=0x4bf035 github.com/cockroachdb/pebble.(*deletionPacer).limit(0xc00b37e6f0, 0x2008fa, {0x4618e7?, 0x28?, 0xa?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:91 +0x22e fp=0xc0050e5e88 sp=0xc0050e5e30 pc=0x1161fee github.com/cockroachdb/pebble.(*deletionPacer).maybeThrottle(0xc00b37e6f0, 0xc000a74940?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/pacer.go:111 +0x3f fp=0xc0050e5ec8 sp=0xc0050e5e88 pc=0x116209f github.com/cockroachdb/pebble.(*DB).paceAndDeleteObsoleteFiles(0xc002a2cd80, 0xc00296b6c0?, {0xc007ea4000?, 0x38, 0xc0106cefb8?}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3130 +0x2b6 fp=0xc0050e5fa8 sp=0xc0050e5ec8 pc=0x110c436 github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles.func5() github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x36 fp=0xc0050e5fe0 sp=0xc0050e5fa8 pc=0x110bf96 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0050e5fe8 sp=0xc0050e5fe0 pc=0x4c2401 created by github.com/cockroachdb/pebble.(*DB).doDeleteObsoleteFiles github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:3111 +0x84a goroutine 498327 [select]: runtime.gopark(0xc00dcb9208?, 0x2?, 0x40?, 0x9d?, 0xc00dcb9204?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00dcb9090 sp=0xc00dcb9070 pc=0x4907d6 runtime.selectgo(0xc00dcb9208, 0xc00dcb9200, 0x10?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00dcb91d0 sp=0xc00dcb9090 pc=0x4a071c google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc012404360) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x7c fp=0xc00dcb9238 sp=0xc00dcb91d0 pc=0xbf9b7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc03bf94c60, {0x51b9a20?, 0xc0130d3480}, 0x6248888?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb fp=0xc00dcb9340 sp=0xc00dcb9238 pc=0xc2c3bb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5?) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 fp=0xc00dcb9370 sp=0xc00dcb9340 pc=0xc2b8e5 google.golang.org/grpc.(*clientStream).withRetry(0xc012404000, 0xc00dcb9448, 0xc00dcb9418) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 fp=0xc00dcb93c8 sp=0xc00dcb9370 pc=0xc2a3b3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc012404000, {0x51b9a20?, 0xc0130d3480?}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f fp=0xc00dcb9478 sp=0xc00dcb93c8 pc=0xc2b61f google.golang.org/grpc.invoke({0x624a948?, 0xc030ec3bc0?}, {0x53d9bbc?, 0x2?}, {0x52f0080, 0xc031df1e00}, {0x51b9a20, 0xc0130d3480}, 0x0?, {0xc003ca8000, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 fp=0xc00dcb94e0 sp=0xc00dcb9478 pc=0xc0a637 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc030ec3bc0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc031df1e00}, {0x51b9a20, 0xc0130d3480}, 0x4cd150cc3806599e?, 0x568c070, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:247 +0x384 fp=0xc00dcb95d0 sp=0xc00dcb94e0 pc=0x1648b84 google.golang.org/grpc.(*ClientConn).Invoke(0x39caeaccd7fd01bf?, {0x624a948?, 0xc030ec3bc0?}, {0x53d9bbc?, 0x0?}, {0x52f0080?, 0xc031df1e00?}, {0x51b9a20?, 0xc0130d3480?}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 fp=0xc00dcb9670 sp=0xc00dcb95d0 pc=0xc0a483 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043276490, {0x624a948, 0xc030ec3bc0}, 0x0?, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9334 +0xc9 fp=0xc00dcb96f0 sp=0xc00dcb9670 pc=0xdcf7a9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x629d930?, 0xc043276490?}}, {0x624a948, 0xc030ec3bc0}, 0xc031df1e00, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:329 +0x171 fp=0xc00dcb97c8 sp=0xc00dcb96f0 pc=0x1671a71 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.(*TracingInternalClient).Batch(0xc88d830?, {0x624a948?, 0xc030ec3bc0?}, 0x624a948?, {0x0?, 0xc005751cd0?, 0xc00c88d928?}) :1 +0x65 fp=0xc00dcb9818 sp=0xc00dcb97c8 pc=0x1671b45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc004d8ccc0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00dcb9ab8 sp=0xc00dcb9818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc004d8ccc0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de82c638, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00dcb9bf0 sp=0xc00dcb9ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de82c638, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00dcba338 sp=0xc00dcb9bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de82c638, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00dcbac38 sp=0xc00dcba338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de82c638, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00dcbb4a0 sp=0xc00dcbac38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76de82c638, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00dcbb8c0 sp=0xc00dcbb4a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00dcbb9f8 sp=0xc00dcbb8c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc030ec2fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00dcbbc00 sp=0xc00dcbb9f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc030ec2fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00dcbbf30 sp=0xc00dcbbc00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00dcbbfe0 sp=0xc00dcbbf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00dcbbfe8 sp=0xc00dcbbfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498228 [select]: runtime.gopark(0xc00f0c8c78?, 0x3?, 0x0?, 0x30?, 0xc00f0c8a9e?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00f0c8900 sp=0xc00f0c88e0 pc=0x4907d6 runtime.selectgo(0xc00f0c8c78, 0xc00f0c8a98, 0x1?, 0x0, 0xc0134d2360?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00f0c8a40 sp=0xc00f0c8900 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForQuery(0xc004959320, {0x624a948, 0xc0134d2360}, 0xc04a4f6d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:805 +0x585 fp=0xc00f0c8cb8 sp=0xc00f0c8a40 pc=0x19bb485 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc00b56cf00, {0x624a948, 0xc0134d2360}, {0x0, {0x17600a76e28c08e2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:358 +0x175 fp=0xc00f0c8cf8 sp=0xc00f0c8cb8 pc=0x1cbd995 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00b56cf00, {0x624a948, 0xc0134d2360}, 0xc006c012c0, 0xc00f0c9298?) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:251 +0x1fc fp=0xc00f0c9168 sp=0xc00f0c8cf8 pc=0x1cbcd7c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x4bad4a0?, {0x624a948, 0xc0134d2360}, 0xc00f0c9350?, {0x0, {0x17600a76e28c08e2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:223 +0x2cc fp=0xc00f0c92a8 sp=0xc00f0c9168 pc=0x1cbca2c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003861800, {0x624a948, 0xc0134d2360}, 0xc00a5af680, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:445 +0x376 fp=0xc00f0c95c8 sp=0xc00f0c92a8 pc=0x1e08df6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003861800, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00f0c9990 sp=0xc00f0c95c8 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00f0ca318 sp=0xc00f0c9990 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00f0ca560 sp=0xc00f0ca318 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc0134d2300}, {0x7f27639619e0?}, 0xc00a5af590) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00f0cabd0 sp=0xc00f0ca560 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0134d2270}, 0xc00a5af590) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00f0cac78 sp=0xc00f0cabd0 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc0134d2270?}, {0x52f0080?, 0xc00a5af590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00f0caca8 sp=0xc00f0cac78 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0134d2270}, {0x52f0080, 0xc00a5af590}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00f0cafd8 sp=0xc00f0caca8 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc0134d2270?}, {0x52f0080?, 0xc00a5af590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00f0cb018 sp=0xc00f0cafd8 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0134d2270}, {0x52f0080, 0xc00a5af590}, 0xc00f0cb0b0?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00f0cb048 sp=0xc00f0cb018 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc0134d2270?}, {0x52f0080?, 0xc00a5af590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00f0cb088 sp=0xc00f0cb048 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0134d2270?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00f0cb0c0 sp=0xc00f0cb088 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0134d2270}, {0xc006b6de00?, 0x0?}, 0xc00f0cb178) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00f0cb138 sp=0xc00f0cb0c0 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0134d2270?}, {0x52f0080?, 0xc00a5af590?}, 0xc00b0fb100?, 0x0?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00f0cb1b0 sp=0xc00f0cb138 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc0134d2270?}, {0x52f0080?, 0xc00a5af590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00f0cb1f0 sp=0xc00f0cb1b0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc0134d2270?}, {0xc0134d2270?, 0x4?}, {0x52f0080?, 0xc00a5af590?}, {0x51b9a20?, 0xc010b7b480}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00f0cb230 sp=0xc00f0cb1f0 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc0134d2270}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a5af590}, {0x51b9a20, 0xc010b7b480}, 0xc00f0cb3a8?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00f0cb320 sp=0xc00f0cb230 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc0134d2270}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc00a5af590}, {0x51b9a20, 0xc010b7b480}, 0xc00f0cb488?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00f0cb3b8 sp=0xc00f0cb320 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc0134d2210}, 0xc00a5af590, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00f0cb438 sp=0xc00f0cb3b8 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xc00b0fb4d8?, {0x624a948?, 0xc0134d2210?}, 0xc00b0fb538?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00f0cb4c0 sp=0xc00f0cb438 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00d118780, {_, _}, _, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00f0cb760 sp=0xc00f0cb4c0 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00d118780, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00f0cb898 sp=0xc00f0cb760 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00f0cbfe0 sp=0xc00f0cb898 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00f0cc8e0 sp=0xc00f0cbfe0 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00f0cd148 sp=0xc00f0cc8e0 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x17600a76e28c1f16, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00f0cd568 sp=0xc00f0cd148 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00f0cd6a0 sp=0xc00f0cd568 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00742c990, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:1016 +0x156 fp=0xc00f0cd7c8 sp=0xc00f0cd6a0 pc=0x16ae696 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(0x4b6f420?, {_, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:999 +0x85 fp=0xc00f0cd8e8 sp=0xc00f0cd7c8 pc=0x16ae4e5 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send-fm({0x624a948?, _}, {{{0x17600a76e28c08e2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, {0x0, ...}, ...}, ...}) :1 +0x85 fp=0xc00f0cd9f8 sp=0xc00f0cd8e8 pc=0x16bc3e5 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x624a948, 0xc01502bcb0}, 0x0?, 0xc016881080) github.com/cockroachdb/cockroach/pkg/kv/db.go:855 +0x119 fp=0xc00f0cdbf0 sp=0xc00f0cd9f8 pc=0x16adcf9 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00742c990, {0x624a948, 0xc01502bcb0}, 0x4596bd?) github.com/cockroachdb/cockroach/pkg/kv/db.go:878 +0x74 fp=0xc00f0cdc38 sp=0xc00f0cdbf0 pc=0x16adeb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc004959320, {0x624a948, 0xc01502bcb0}, {{0x1d, 0x74, 0x91, 0xa, 0x95, 0x83, 0x43, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:928 +0x1b0 fp=0xc00f0cdc88 sp=0xc00f0cdc38 pc=0x19bc450 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).startQueryPusherTxn.func1({0x624a948, 0xc01502bcb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:854 +0x238 fp=0xc00f0cdf30 sp=0xc00f0cdc88 pc=0x19bbe78 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00f0cdfe0 sp=0xc00f0cdf30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00f0cdfe8 sp=0xc00f0cdfe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498483 [select]: runtime.gopark(0xc01750fc18?, 0x3?, 0xd2?, 0xf9?, 0xc01750fb3a?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc01750f8f0 sp=0xc01750f8d0 pc=0x4907d6 runtime.selectgo(0xc01750fc18, 0xc01750fb34, 0xc00511d500?, 0x0, 0x18?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc01750fa30 sp=0xc01750f8f0 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).resolveIntents(0xc0005593b0, {0x624a948, 0xc00564c4e0}, {0x6221560, 0xc003ffd200}, {0x46?, {0xc01750fe01?, 0x1750fe60?, 0xc0?}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:948 +0x909 fp=0xc01750fda0 sp=0xc01750fa30 pc=0x1ca8ee9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0005593b0, {0x624a948?, 0xc00564c4e0?}, 0x164, 0xc003ffd200, 0x0?, 0xc01283e0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:777 +0xd1 fp=0xc01750fe70 sp=0xc01750fda0 pc=0x1ca77b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x624a948, 0xc00564c4e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:583 +0xe5 fp=0xc01750ff30 sp=0xc01750fe70 pc=0x1ca6165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc01750ffe0 sp=0xc01750ff30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc01750ffe8 sp=0xc01750ffe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498500 [semacquire]: runtime.gopark(0xae1b77a68b4?, 0x0?, 0x0?, 0x0?, 0x2?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fd95708 sp=0xc00fd956e8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0x34?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc00fd95770 sp=0xc00fd95708 pc=0x4a17de sync.runtime_SemacquireMutex(0x0?, 0xe0?, 0x10b86d8?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc00fd957a0 sp=0xc00fd95770 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc0005ee8b0, 0x7f271e737420?, 0xc00cde35c0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c fp=0xc00fd957f8 sp=0xc00fd957a0 pc=0x10a35ac github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x10c38fa?, 0xc0109fc000?, 0xc00fd95888?, 0x4ccece?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c fp=0xc00fd95840 sp=0xc00fd957f8 pc=0x10a571c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0109fc000, {0x6?, 0x6?}, 0x0, 0xc00628f178, 0xc00d99ed10) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2767 +0x79 fp=0xc00fd95940 sp=0xc00fd95840 pc=0x10caed9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:583 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc00628ef00, 0x90?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:498 +0x28d fp=0xc00fd959e0 sp=0xc00fd95940 pc=0x10c3f6d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc00628ef00, {0xc00a4e8990, 0x1e, 0x21}, 0xc00a4e8a20?, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:790 +0x3bd fp=0xc00fd95a90 sp=0xc00fd959e0 pc=0x10c4c3d github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x0?, {0xc00a4e8a20?, 0x0?, 0x0?}, {0xc00a4e8990?, 0x1e?, 0xc00548bce0?}, 0x30?, 0x4?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:890 +0x305 fp=0xc00fd95af8 sp=0xc00fd95a90 pc=0x10c5285 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc00d9a02e0?, {0xc00a4e8a20?, 0x1?, 0x21?}, {0xc00a4e8990?, 0x18?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:826 +0x2b fp=0xc00fd95b48 sp=0xc00fd95af8 pc=0x10c4f0b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00d9a02e0, {0xc00a4e8a20, 0x1e, 0x21}, {0xc00a4e8990, 0x1e, 0x21}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:674 +0x137 fp=0xc00fd95b98 sp=0xc00fd95b48 pc=0x114bad7 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00d99ef20, {0xc00a4e8990?, 0x0?, 0x0?}, 0x0?, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:906 +0xd8 fp=0xc00fd95cc8 sp=0xc00fd95b98 pc=0x1152218 github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00d99ef20, {0xc00a4e8a20?, 0x0?, 0x0?}, {0xc00a4e8990?, 0x0?, 0x0?}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:982 +0x9d fp=0xc00fd95d08 sp=0xc00fd95cc8 pc=0x115285d github.com/cockroachdb/pebble.(*lazyCombinedIter).SeekPrefixGE(0xc00d99ed88, {0xc00a4e8a20?, 0x0?, 0x21?}, {0xc00a4e8990, 0x1e, 0x21}, 0x0?) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/range_keys.go:581 +0x53 fp=0xc00fd95d60 sp=0xc00fd95d08 pc=0x1163ed3 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00d99ea00, {0xc00a4e8990, 0x1e, 0x21}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1326 +0x59d fp=0xc00fd95df8 sp=0xc00fd95d60 pc=0x113e99d github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc00e53b608, {{0xc0082d0000, 0x1d, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:331 +0xd6 fp=0xc00fd95e48 sp=0xc00fd95df8 pc=0x1862d16 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc002f41b00, {0x624a948, 0xc0400317d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:462 +0xbf fp=0xc00fd95eb0 sp=0xc00fd95e48 pc=0x186a01f github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x624a948, 0xc0400317d0}, {0x62b4fe0, 0xc00e53b608}, {0xc0082d0000, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1031 +0x3fd fp=0xc00fd96480 sp=0xc00fd95eb0 pc=0x1816abd github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x624a948, 0xc0400317d0}, {0x628f760?, 0xc00e53b600?}, {0xc0082d0000, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:986 +0x1a6 fp=0xc00fd96600 sp=0xc00fd96480 pc=0x1816446 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x624a948?, 0xc0400317d0?}, {0x628f760?, 0xc00e53b600?}, {0xc0082d0000?, 0xc00fd967a0?, 0x4618e7?}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:810 +0xbe fp=0xc00fd966f0 sp=0xc00fd96600 pc=0x1815a1e github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0?, {0x624a948, 0xc0400317d0}, {0x628f760, 0xc00e53b600}, {0xfe, 0xf5, 0xf5, 0x20, 0x9, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:95 +0xd3 fp=0xc00fd967b0 sp=0xc00fd966f0 pc=0x1c96133 github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0xfe, 0xf5, 0xf5, 0x20, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:2034 +0xbc fp=0xc00fd96998 sp=0xc00fd967b0 pc=0x1d8fb1c github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc010617e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:206 +0x36f fp=0xc00fd98808 sp=0xc00fd96998 pc=0x1dc43af github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, 0xc00fd99278, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:318 +0x3f3 fp=0xc00fd98e78 sp=0xc00fd98808 pc=0x1e05ef3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc006f10000, {0x624a948, 0xc0400317d0}, 0xc010617e00, 0xc015d26a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:107 +0x554 fp=0xc00fd99b08 sp=0xc00fd98e78 pc=0x1e04d14 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc0400317d0}, 0xc010617e00, 0x5683ba8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00fd99e28 sp=0xc00fd99b08 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a764d68fcff, 0x0, 0x0}, 0x0, {0x17600a76e3aba611, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:176 +0x6a5 fp=0xc00fd9a1f0 sp=0xc00fd99e28 pc=0x1e07a85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a764d68fcff, 0x0, 0x0}, 0x0, {0x17600a76e3aba611, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00fd9ab78 sp=0xc00fd9a1f0 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00a1ed740?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e3aba611, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00fd9adc0 sp=0xc00fd9ab78 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc040031770}, {0x0?}, 0xc010617d10) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00fd9b430 sp=0xc00fd9adc0 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc0400316b0}, 0xc010617d10) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00fd9b4d8 sp=0xc00fd9b430 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x624a948, 0xc0400316b0}, {0x52f0080?, 0xc010617d10}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9619 +0x78 fp=0xc00fd9b518 sp=0xc00fd9b4d8 pc=0xdd0d38 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc0400316b0}, {0x52f0080, 0xc010617d10}, 0xc003402d80, 0xc0083568d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00fd9b848 sp=0xc00fd9b518 pc=0x1647414 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0400316b0?}, {0x52f0080?, 0xc010617d10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b fp=0xc00fd9b888 sp=0xc00fd9b848 pc=0xc1f1db github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc0400316b0}, {0x52f0080, 0xc010617d10}, 0xc00fd9b920?, 0xc03bfb2700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00fd9b8b8 sp=0xc00fd9b888 pc=0x16577c3 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0400316b0?}, {0x52f0080?, 0xc010617d10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fd9b8f8 sp=0xc00fd9b8b8 pc=0xc1f203 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc0400316b0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00fd9b930 sp=0xc00fd9b8f8 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc0400316b0}, {0x203000?, 0x40?}, 0xc00fd9b9e8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00fd9b9a8 sp=0xc00fd9b930 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc0400316b0?}, {0x52f0080?, 0xc010617d10?}, 0x5003b00?, 0x7f278cfaa101?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00fd9ba20 sp=0xc00fd9b9a8 pc=0x1657a35 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x624a948?, 0xc0400316b0?}, {0x52f0080?, 0xc010617d10?}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 fp=0xc00fd9ba60 sp=0xc00fd9ba20 pc=0xc1f203 google.golang.org/grpc.chainUnaryInterceptors.func1({0x624a948, 0xc0400316b0}, {0x52f0080, 0xc010617d10}, 0xc003402d80, 0xc0083568d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b fp=0xc00fd9bab0 sp=0xc00fd9ba60 pc=0xc1f12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x52b4780?, 0xc00a1cf500}, {0x624a948, 0xc0400316b0}, 0xc011f07380, 0xc001a44d40) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9621 +0x138 fp=0xc00fd9bb08 sp=0xc00fd9bab0 pc=0xdd0bf8 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00512ca20, 0xc000acee70, 0x8ffb4e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcfe fp=0xc00fd9be48 sp=0xc00fd9bb08 pc=0xc1ff5e google.golang.org/grpc.(*Server).handleStream(0xc00296b6c0, {0x6294e00, 0xc0038e3380}, 0xc00512ca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2f fp=0xc00fd9bf68 sp=0xc00fd9be48 pc=0xc2462f google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 fp=0xc00fd9bfe0 sp=0xc00fd9bf68 pc=0xc1dab8 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fd9bfe8 sp=0xc00fd9bfe0 pc=0x4c2401 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x28a goroutine 498182 [select]: runtime.gopark(0xc007cebfa8?, 0x2?, 0x4?, 0x30?, 0xc007cebf74?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007cebdc8 sp=0xc007cebda8 pc=0x4907d6 runtime.selectgo(0xc007cebfa8, 0xc007cebf70, 0xc006ff2ea0?, 0x0, 0x1054cca?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc007cebf08 sp=0xc007cebdc8 pc=0x4a071c github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:71 +0xe5 fp=0xc007cebfe0 sp=0xc007cebf08 pc=0x1058345 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007cebfe8 sp=0xc007cebfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:66 +0x5d goroutine 498458 [select]: runtime.gopark(0xc00fd85300?, 0x3?, 0x1?, 0x0?, 0xc00fd84fca?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc00fd84d50 sp=0xc00fd84d30 pc=0x4907d6 runtime.selectgo(0xc00fd85300, 0xc00fd84fc4, 0x17600a76e28bfed8?, 0x0, 0x1?, 0x1) GOROOT/src/runtime/select.go:328 +0x7bc fp=0xc00fd84e90 sp=0xc00fd84d50 pc=0x4a071c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006f10000, {0x624a948, 0xc011c547e0}, 0xc0081db680, 0xc00c4f34a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:200 +0x8d1 fp=0xc00fd85600 sp=0xc00fd84e90 pc=0x1e14831 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006f10000, {0x624a948, 0xc011c547e0}, 0xc0081db680, 0x5683bb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:481 +0x3ce fp=0xc00fd85920 sp=0xc00fd85600 pc=0x1e08e4e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006f10000, {_, _}, {{{0x17600a76e28bfed8, 0x0, 0x0}, 0xc00413aa10, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:180 +0x70a fp=0xc00fd85ce8 sp=0xc00fd85920 pc=0x1e07aea github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SendWithWriteBytes(0xc00679ea80, {_, _}, {{{0x17600a76e28bfed8, 0x0, 0x0}, 0xc00413aa10, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:205 +0x8a9 fp=0xc00fd86670 sp=0xc00fd85ce8 pc=0x1e54b49 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).SendWithWriteBytes(0xc00ba829e0?, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:203 +0xee fp=0xc00fd868b8 sp=0xc00fd86670 pc=0x1e63e8e github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc00a1cf500, {0x624a948, 0xc011c54630}, {0xc006a4a780?}, 0xc0081db590) github.com/cockroachdb/cockroach/pkg/server/node.go:1108 +0x578 fp=0xc00fd86f28 sp=0xc00fd868b8 pc=0x39bfd38 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc00a1cf500, {0x624a948, 0xc011c542d0}, 0xc0081db590) github.com/cockroachdb/cockroach/pkg/server/node.go:1161 +0xfd fp=0xc00fd86fd0 sp=0xc00fd86f28 pc=0x39c043d github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1({0x624a948?, 0xc011c542d0?}, {0x52f0080?, 0xc0081db590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:724 +0x4b fp=0xc00fd87000 sp=0xc00fd86fd0 pc=0x1659c0b github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ServerInterceptor.func1({0x624a948, 0xc011c542d0}, {0x52f0080, 0xc0081db590}, 0xc0026983c0, 0xc0058f5878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:96 +0x254 fp=0xc00fd87330 sp=0xc00fd87000 pc=0x1647414 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011c542d0?}, {0x52f0080?, 0xc0081db590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00fd87370 sp=0xc00fd87330 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func3({0x624a948, 0xc011c542d0}, {0x52f0080, 0xc0081db590}, 0xc00ba83408?, 0xc0026983e0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:263 +0x83 fp=0xc00fd873a0 sp=0xc00fd87370 pc=0x16577c3 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011c542d0?}, {0x52f0080?, 0xc0081db590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00fd873e0 sp=0xc00fd873a0 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1.1({0x624a948?, 0xc011c542d0?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:232 +0x39 fp=0xc00fd87418 sp=0xc00fd873e0 pc=0x1657af9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc002a40360, {0x624a948, 0xc011c542d0}, {0xc003ca8000?, 0x3?}, 0xc00ba834d0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:341 +0xd1 fp=0xc00fd87490 sp=0xc00fd87418 pc=0x10542d1 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func1({0x624a948?, 0xc011c542d0?}, {0x52f0080?, 0xc0081db590?}, 0x51b9a20?, 0xc00d502100?) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:230 +0x95 fp=0xc00fd87508 sp=0xc00fd87490 pc=0x1657a35 github.com/cockroachdb/cockroach/pkg/rpc.bindUnaryServerInterceptorToHandler.func1({0x624a948?, 0xc011c542d0?}, {0x52f0080?, 0xc0081db590?}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:795 +0x3a fp=0xc00fd87548 sp=0xc00fd87508 pc=0x1659e1a github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func2({0x624a948?, 0xc011c542d0?}, {0xc011c542d0?, 0x4?}, {0x52f0080?, 0xc0081db590?}, {0x51b9a20?, 0xc00d502180}, 0x203004?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:734 +0x54 fp=0xc00fd87588 sp=0xc00fd87548 pc=0x1659a94 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x624a948, 0xc011c542d0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081db590}, {0x51b9a20, 0xc00d502180}, 0xc00ba83700?, 0xc005a00540, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:226 +0x5c3 fp=0xc00fd87678 sp=0xc00fd87588 pc=0x1648dc3 github.com/cockroachdb/cockroach/pkg/rpc.getChainUnaryInvoker.func1({0x624a948, 0xc011c542d0}, {0x53d9bbc, 0x21}, {0x52f0080, 0xc0081db590}, {0x51b9a20, 0xc00d502180}, 0xc00ba837e0?, {0x0, ...}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:879 +0x13e fp=0xc00fd87710 sp=0xc00fd87678 pc=0x165a71e github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func3({0x624a948, 0xc011c54240}, 0xc0081db590, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:764 +0xf8 fp=0xc00fd87790 sp=0xc00fd87710 pc=0x16599d8 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:887 github.com/cockroachdb/cockroach/pkg/rpc.(*internalClientAdapter).Batch(0xba83830?, {0x624a948?, 0xc011c54240?}, 0x624a948?, {0x0?, 0xc006922000?, 0xc006922000?}) :1 +0x8f fp=0xc00fd87818 sp=0xc00fd87790 pc=0x1668a8f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0134e71a0, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:211 +0x17e fp=0xc00fd87ab8 sp=0xc00fd87818 pc=0x19daefe github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0134e71a0, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:190 +0x158 fp=0xc00fd87bf0 sp=0xc00fd87ab8 pc=0x19dad18 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2079 +0x110f fp=0xc00fd88338 sp=0xc00fd87bf0 pc=0x19ce2ef github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1623 +0x805 fp=0xc00fd88c38 sp=0xc00fd88338 pc=0x19cade5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1221 +0x433 fp=0xc00fd894a0 sp=0xc00fd88c38 pc=0x19c8053 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002aac500, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x17600a76e28ac3f6, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:845 +0x629 fp=0xc00fd898c0 sp=0xc00fd894a0 pc=0x19c5fe9 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00742c9f8, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xda fp=0xc00fd899f8 sp=0xc00fd898c0 pc=0x16a9a7a github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x624a948, 0xc0032103c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xeb fp=0xc00fd89c00 sp=0xc00fd899f8 pc=0x1ca158b github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x624a948, 0xc0032103c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x214 fp=0xc00fd89f30 sp=0xc00fd89c00 pc=0x1ca0bf4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:489 +0x146 fp=0xc00fd89fe0 sp=0xc00fd89f30 pc=0x1054c46 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00fd89fe8 sp=0xc00fd89fe0 pc=0x4c2401 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:480 +0x43b goroutine 498166 [semacquire]: runtime.gopark(0xae1b72e8c2e?, 0x1?, 0x11?, 0x0?, 0xc02e882000?) GOROOT/src/runtime/proc.go:363 +0xd6 fp=0xc007cedd18 sp=0xc007cedcf8 pc=0x4907d6 runtime.goparkunlock(...) GOROOT/src/runtime/proc.go:369 runtime.semacquire1(0xc0005ee8cc, 0xb0?, 0x3, 0x0) GOROOT/src/runtime/sema.go:150 +0x1fe fp=0xc007cedd80 sp=0xc007cedd18 pc=0x4a17de sync.runtime_SemacquireMutex(0xc007cedf80?, 0x85?, 0xc0147999ec?) GOROOT/src/runtime/sema.go:77 +0x25 fp=0xc007ceddb0 sp=0xc007cedd80 pc=0x4be125 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:71 github.com/cockroachdb/pebble/internal/cache.(*shard).Delete(0xc0005ee8b0, 0x0?, 0x0?, 0x0?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:229 +0x65 fp=0xc007cede30 sp=0xc007ceddb0 pc=0x10a3ee5 github.com/cockroachdb/pebble/internal/cache.(*Cache).Delete(0xc00ab3a000?, 0x0?, 0x80?, 0xc01575cc00?) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:755 +0x3c fp=0xc007cede78 sp=0xc007cede30 pc=0x10a583c github.com/cockroachdb/pebble/sstable.(*Writer).writeCompressedBlock(0xc010822a80, {0xc012f70000, 0x20033, 0x40000}, {0xc00125ad20, 0xc0061b8900?, 0x78}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:1482 +0x74 fp=0xc007cedeb0 sp=0xc007cede78 pc=0x10dfef4 github.com/cockroachdb/pebble/sstable.(*writeQueue).performWrite(0xc00c984580, 0xc00f7f6070) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:68 +0x85 fp=0xc007cedf90 sp=0xc007cedeb0 pc=0x10d76c5 github.com/cockroachdb/pebble/sstable.(*writeQueue).runWorker(0xc00c984580) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:109 +0x85 fp=0xc007cedfc8 sp=0xc007cedf90 pc=0x10d7d45 github.com/cockroachdb/pebble/sstable.newWriteQueue.func1() github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0x26 fp=0xc007cedfe0 sp=0xc007cedfc8 pc=0x10d7606 runtime.goexit() GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc007cedfe8 sp=0xc007cedfe0 pc=0x4c2401 created by github.com/cockroachdb/pebble/sstable.newWriteQueue github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0xd6