Flaky Test: Test/HandleListenerUpdate_ErrorUpdate

[arjan-bal]

panic: test timed out after 7m0s
	running tests:
		Test (6m52s)
		Test/HandleListenerUpdate_ErrorUpdate (6m52s)

goroutine 3874 [running]:
testing.(*M).startAlarm.func1()
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:2484 +0x605
created by time.goFunc
	/opt/hostedtoolcache/go/1.24.3/x64/src/time/sleep.go:215 +0x45

goroutine 1 [chan receive, 6 minutes]:
testing.(*T).Run(0xc001c4ea80, {0x19a81df, 0x4}, 0x1a072c0)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1859 +0x91e
testing.runTests.func1(0xc001c4ea80)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:2279 +0x86
testing.tRunner(0xc001c4ea80, 0xc000341ae0)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1792 +0x226
testing.runTests(0xc000011188, {0x264ee00, 0x2, 0x2}, {0x4?, 0x0?, 0x266fdc0?})
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:2277 +0x96d
testing.(*M).Run(0xc00058e320)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:2142 +0xeeb
main.main()
	_testmain.go:49 +0x165

goroutine 23 [chan send, 6 minutes]:
google.golang.org/grpc/internal/testutils.(*Channel).Send(...)
	/home/runner/work/grpc-go/grpc-go/internal/testutils/channel.go:37
google.golang.org/grpc/xds.s.TestHandleListenerUpdate_ErrorUpdate.func2({0x1b77b70, 0xc0002bdec0}, {0x7b7c05?, {0x1b620c0?, 0xc0006a40c0?}})
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:610 +0x1b7
google.golang.org/grpc/xds.(*GRPCServer).Serve.func1({0x1b77b70, 0xc0002bdec0}, 0x2, {0x1b620c0, 0xc0006a40c0})
	/home/runner/work/grpc-go/grpc-go/xds/server.go:207 +0x8f
google.golang.org/grpc/xds/internal/server.(*listenerWrapper).switchModeLocked(0xc000426480, 0x2, {0x1b620c0, 0xc0006a40c0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/server/listener_wrapper.go:377 +0x223
google.golang.org/grpc/xds/internal/server.(*ldsWatcher).ResourceError(0xc00056a0e0, {0x1b620c0, 0xc0006a40c0}, 0xc0009e40d8)
	/home/runner/work/grpc-go/grpc-go/xds/internal/server/listener_wrapper.go:443 +0x40f
google.golang.org/grpc/xds/internal/xdsclient/xdsresource.(*delegatingListenerWatcher).ResourceError(0xc00058a320, {0x1b620c0, 0xc0006a40c0}, 0xc0009e40d8)
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/xdsresource/listener_resource_type.go:172 +0x5c
google.golang.org/grpc/xds/internal/xdsclient.(*wrappingWatcher).ResourceError(0xc00056a120, {0x1b62c40, 0xc0009e4060}, 0xc0009e40d8)
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/clientimpl_watchers.go:37 +0x12c
google.golang.org/grpc/xds/internal/xdsclient.(*authority).handleADSResourceUpdate.func3({0xc0002bd860?, 0xc0004b9ea8?})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/authority.go:379 +0x67
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000476410, {0x1b7d730, 0xc000582000})
	/home/runner/work/grpc-go/grpc-go/internal/grpcsync/callback_serializer.go:94 +0x266
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3853
	/home/runner/work/grpc-go/grpc-go/internal/grpcsync/callback_serializer.go:52 +0x[205](https://github.com/grpc/grpc-go/actions/runs/14890651708/job/41821604279?pr=8290#step:8:206)

goroutine 3853 [chan receive, 6 minutes]:
google.golang.org/grpc/xds/internal/xdsclient.(*clientImpl).close(0xc00055abd0)
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/clientimpl.go:[209](https://github.com/grpc/grpc-go/actions/runs/14890651708/job/41821604279?pr=8290#step:8:210) +0x4be
google.golang.org/grpc/xds/internal/xdsclient.(*Pool).clientRefCountedClose(0xc000506150, {0x19aaaaa, 0x7})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/pool.go:[211](https://github.com/grpc/grpc-go/actions/runs/14890651708/job/41821604279?pr=8290#step:8:212) +0x112
google.golang.org/grpc/xds/internal/xdsclient.(*Pool).newRefCounted.func2()
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/pool.go:261 +0x47
sync.OnceFunc.func1()
	/opt/hostedtoolcache/go/1.24.3/x64/src/sync/oncefunc.go:27 +0x7c
sync.(*Once).doSlow(0xc00059e884, 0xc000137320)
	/opt/hostedtoolcache/go/1.24.3/x64/src/sync/once.go:78 +0xe2
sync.(*Once).Do(0xc00059e884, 0xc000137320)
	/opt/hostedtoolcache/go/1.24.3/x64/src/sync/once.go:69 +0x45
sync.OnceFunc.func2()
	/opt/hostedtoolcache/go/1.24.3/x64/src/sync/oncefunc.go:32 +0x4b
google.golang.org/grpc/xds.(*GRPCServer).Stop(0xc000441bc0)
	/home/runner/work/grpc-go/grpc-go/xds/server.go:224 +0x93
google.golang.org/grpc/xds.s.TestHandleListenerUpdate_ErrorUpdate({{}}, 0xc001c4ee00)
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:666 +0x1411
google.golang.org/grpc/internal/grpctest.RunSubTests.func1(0xc001c4ee00)
	/home/runner/work/grpc-go/grpc-go/internal/grpctest/grpctest.go:127 +0x10b
testing.tRunner(0xc001c4ee00, 0xc0005b1980)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1792 +0x[226](https://github.com/grpc/grpc-go/actions/runs/14890651708/job/41821604279?pr=8290#step:8:227)
created by testing.(*T).Run in goroutine 3852
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1851 +0x8f3

goroutine 3852 [chan receive, 6 minutes]:
testing.(*T).Run(0xc001c4ec40, {0x1698d86, 0x20}, 0xc0005b1980)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1859 +0x91e
google.golang.org/grpc/internal/grpctest.RunSubTests(0xc001c4ec40, {0x1b77b20, 0x26912c0})
	/home/runner/work/grpc-go/grpc-go/internal/grpctest/grpctest.go:119 +0x34b
google.golang.org/grpc/xds.Test(0xc001c4ec40)
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:65 +0x35
testing.tRunner(0xc001c4ec40, 0x1a072c0)
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1792 +0x226
created by testing.(*T).Run in goroutine 1
	/opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1851 +0x8f3

goroutine 3855 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7fc154d4caa8, 0x72)
	/opt/hostedtoolcache/go/1.24.3/x64/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc00062afa0, 0x72, 0x0)
	/opt/hostedtoolcache/go/1.24.3/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.24.3/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00062af80)
	/opt/hostedtoolcache/go/1.24.3/x64/src/internal/poll/fd_unix.go:620 +0x505
net.(*netFD).accept(0xc00062af80)
	/opt/hostedtoolcache/go/1.24.3/x64/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc000611640)
	/opt/hostedtoolcache/go/1.24.3/x64/src/net/tcpsock_posix.go:159 +0x4e
net.(*TCPListener).Accept(0xc000611640)
	/opt/hostedtoolcache/go/1.24.3/x64/src/net/tcpsock.go:380 +0x65
google.golang.org/grpc.(*Server).Serve(0xc0006ac600, {0x1b7b638, 0xc000611640})
	/home/runner/work/grpc-go/grpc-go/server.go:890 +0x7d8
created by google.golang.org/grpc/internal/testutils/xds/e2e.StartManagementServer in goroutine 3853
	/home/runner/work/grpc-go/grpc-go/internal/testutils/xds/e2e/server.go:185 +0x95a

https://github.com/grpc/grpc-go/actions/runs/14890651708/job/41821604279?pr=8290
https://github.com/grpc/grpc-go/actions/runs/14890651708/job/41821604280?pr=8290

[arjan-bal]

@purnesh42H this failed twice for the same PR today. Could these failures be related to any xDS client changes that were merged recently?

[arjan-bal]

https://github.com/grpc/grpc-go/actions/runs/15109206128/job/42464635601?pr=8341

[arjan-bal]

https://github.com/grpc/grpc-go/actions/runs/15256232151/job/42904441613?pr=8360

[arjan-bal]

https://github.com/grpc/grpc-go/actions/runs/15296375682/job/43026365208?pr=8365

[arjan-bal]

https://github.com/grpc/grpc-go/actions/runs/15307204328/job/43062811171?pr=8367

[arjan-bal]

There were changes made to this test in #7977

[arjan-bal]

I checked the logs on forge, the serving mode callback is called 3 times with the same update:

grpc/xds/server_test.go:609: Server mode change callback invoked for listener "127.0.0.1:43883" with mode "NOT_SERVING" and error [xDS node id: 0fa7209c-6282-4495-b754-a315aa8b5271]: unsupported field 'listener_filters' contains 1 entries

The test only reads one update from the testutils.Channel and the channel has a capacity of 1, so sending the third update on the channel blocks indefinitely.

[arjan-bal]

Another problem in the test is that it's using a short context to wait an update that is expected to arrive:

grpc-go/xds/server_test.go

Lines 661 to 665 in ec4810c

	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
	defer sCancel()
	if _, err := modeChangeCh.Receive(sCtx); err == context.DeadlineExceeded {
	t.Fatal("Serving mode did not change when expected to change")
	}

We should use the regular context instead.

[arjan-bal]

https://github.com/grpc/grpc-go/actions/runs/15339040694/job/43161639292?pr=8367

[Pranjali-2501]

https://github.com/grpc/grpc-go/actions/runs/15411638796/job/43364748821?pr=8278

[eshitachandwani]

https://github.com/grpc/grpc-go/actions/runs/15495281521/job/43630399427?pr=8384

[arjan-bal]

https://github.com/grpc/grpc-go/actions/runs/15552025142/job/43784328282?pr=8390