pkg/ccl/schemachangerccl/sctestbackupccl/sctestbackupccl_test: TestBackupSuccessMixedVersion_multiregion_add_column_multiple_regional_by_row failed

[github-actions]

pkg/ccl/schemachangerccl/sctestbackupccl/sctestbackupccl_test.TestBackupSuccessMixedVersion_multiregion_add_column_multiple_regional_by_row failed on master @ 87d077aff540406ce47db452d1fa9a1d810cd3f9:

github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3({0xc00c7a5bc0?, 0x867d5f0?})
	pkg/server/start_listen.go:216 +0x26
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x867d5f0, 0xc00f3083c0})
	pkg/util/stop/stopper.go:438 +0x8a
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 2107359
	pkg/util/stop/stopper.go:436 +0x137

goroutine 2111636 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x867cb98, 0xc00ff2f500})
	pkg/kv/kvserver/queue.go:852 +0xe9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x867cb98, 0xc00ff2f500})
	pkg/util/stop/stopper.go:438 +0x8a
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 2111490
	pkg/util/stop/stopper.go:436 +0x137

goroutine 2111437 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*peer[...]).runHeartbeatUntilFailure(0x8740020, {0x867cb98, 0xc010b26f00}, 0xc00e333730)
	pkg/rpc/peer.go:549 +0x21f
github.com/cockroachdb/cockroach/pkg/rpc.(*peer[...]).runOnce(0x8740020, {0x867cb98, 0xc010b26f00}, 0xc0018c17c0)
	pkg/rpc/peer.go:424 +0x348
github.com/cockroachdb/cockroach/pkg/rpc.(*peer[...]).run(0x8740020, {0x867cb98, 0xc010b26f00}, 0xc0018c17c0, 0xc00c7a4240)
	pkg/rpc/peer.go:353 +0x1fd
github.com/cockroachdb/cockroach/pkg/rpc.(*peer[...]).launch.func1()
	pkg/rpc/peer.go:314 +0x3d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x867cb98, 0xc010b26f00})
	pkg/util/stop/stopper.go:438 +0x8a
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 2111436
	pkg/util/stop/stopper.go:436 +0x137

goroutine 2111923 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStatsIngester).Start.func1({0x867cb98, 0xc00a1e9ec0})
	pkg/sql/sqlstats/sslocal/sql_stats_ingestor.go:181 +0xc5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x867cb98, 0xc00a1e9ec0})
	pkg/util/stop/stopper.go:438 +0x8a
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 2107359
	pkg/util/stop/stopper.go:436 +0x137

goroutine 2111754 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00f40c820)
	external/org_golang_google_grpc/internal/transport/http2_server.go:1150 +0x1ec
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 2111746
	external/org_golang_google_grpc/internal/transport/http2_server.go:339 +0x1a6e

goroutine 2107853 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSenderCircuitBreakers).gcLoop(0xc00ec9e008, {0x867d5f0, 0xc008d32140})
	pkg/kv/kvclient/kvcoord/dist_sender_circuit_breaker.go:329 +0x273
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func1({0x867d5f0, 0xc008d32140})
	pkg/util/stop/stopper.go:438 +0x8a
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 2107359
	pkg/util/stop/stopper.go:436 +0x137

Parameters:

• attempt=1
• run=1
• shard=34

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/sql-foundations

This test on roachdash | Improve this report!

Jira issue: CRDB-53631

[fqazi]

This looks like the same timeout issue, our attempts to speed it up haven't help. I have one more idea we can try, let me see if that helps improve performance here.