sql: TestTruncateLarge failed

[cockroach-teamcity]

sql.TestTruncateLarge failed with artifacts on master @ 49cbc690503fc8c05251707580469d144629b147:

github.com/cockroachdb/cockroach/pkg/sql/gcjob.waitForGC({0x7e31f80?, 0x402cc66b40?}, 0x40070c5201?, 0x402cd49e00?, 0x402ba9b048?)
	pkg/sql/gcjob/gc_job.go:409 +0x78
github.com/cockroachdb/cockroach/pkg/sql/gcjob.schemaChangeGCResumer.deleteDataAndWaitForGC({_}, {_, _}, {0x4010050000, 0x40074d2e10, {0x4014689cb0, 0x400cbe0140, 0x4014689ca0, 0x400d0aab20}, {{0x4026c066f0, ...}, ...}, ...}, ...)
	pkg/sql/gcjob/gc_job.go:397 +0x144
github.com/cockroachdb/cockroach/pkg/sql/gcjob.schemaChangeGCResumer.Resume({0x402cd49e00}, {0x7e31f80, 0x402cc66b40}, {0x68a9400?, 0x4011928480?})
	pkg/sql/gcjob/gc_job.go:376 +0x248
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2(0x402cd49e00?, 0x0?, 0x4031ad5900, {0x7e238c0?, 0x4011928450?}, {0x7e31f80?, 0x402cc66b40?}, {0x68a9400?, 0x4011928480?})
	pkg/jobs/registry.go:1641 +0xe8
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine(0x400cb84008, {0x7e31f48, 0x405bd00460}, {0x68a9400, 0x4011928480}, {0x7e238c0, 0x4011928450}, 0x402cd49e00, {0x400ad6f840, 0x7}, ...)
	pkg/jobs/registry.go:1642 +0x870
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob(0x400cb84008, {0x7e31f48, 0x405bd00460}, {0x7e238c0, 0x4011928450}, 0x402cd49e00, {0x400ad6f840, 0x7}, {0x401be8e420, 0x17})
	pkg/jobs/adopt.go:415 +0x348
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func1({0x7e33140?, 0x404596bb80?})
	pkg/jobs/adopt.go:268 +0xe0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x7e33140, 0x404596bb80})
	pkg/util/stop/stopper.go:440 +0x7c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 653064
	pkg/util/stop/stopper.go:438 +0xe0

goroutine 405824 [select, 11 minutes, "tags":"nsql?,rnode=?,raddr=127.0.0.1:34941,class=system,rpc"]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0x401b96bde0, {0x7e31f48, 0x4036a5b7c0})
	external/org_golang_google_grpc/internal/grpcsync/callback_serializer.go:83 +0xbc
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 405822
	external/org_golang_google_grpc/internal/grpcsync/callback_serializer.go:55 +0x10c

goroutine 403393 [sync.Cond.Wait, 11 minutes]:
sync.runtime_notifyListWait(0x4007a86fd0, 0x710)
	GOROOT/src/runtime/sema.go:606 +0x140
sync.(*Cond).Wait(0x4007a86fc0)
	GOROOT/src/sync/cond.go:71 +0xa4
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*schedulerShard).processEvents(0x400ea81500, {0x7e33140, 0x400dc97720})
	pkg/kv/kvserver/rangefeed/scheduler.go:476 +0x5dc
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Scheduler).Start.func1({0x7e33140, 0x400dc97720})
	pkg/kv/kvserver/rangefeed/scheduler.go:249 +0xdc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x7e33140, 0x400dc97720})
	pkg/util/stop/stopper.go:440 +0x7c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 403322
	pkg/util/stop/stopper.go:438 +0xe0

goroutine 403565 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0x4007a86910, 0x1e56)
	GOROOT/src/runtime/sema.go:606 +0x140
sync.(*Cond).Wait(0x4007a86900)
	GOROOT/src/sync/cond.go:71 +0xa4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0x402810f880, {0x7e33140, 0x4021c614a0}, {0x7e61c20, 0x4010392408}, 0x4004193008)
	pkg/kv/kvserver/scheduler.go:378 +0x34c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7e33140, 0x4021c614a0}, 0x7e33140?)
	pkg/kv/kvserver/scheduler.go:314 +0xcc
created by github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start in goroutine 403322
	pkg/kv/kvserver/scheduler.go:328 +0x154

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/sql-foundations

This test on roachdash | Improve this report!

Jira issue: CRDB-56308

[yuzefovich]

dup of #156998 - hopefully #157079 will make this test more reasonable