operator panics on watch reconnect — all DeleteFunc handlers missing cache.DeletedFinalStateUnknown handling

[abhi2303237]

Summary

When the watch connection to the API server drops and reconnects, client-go performs
a resync re-list. Any object that was deleted while the connection was down cannot be
delivered as a normal DELETED event — instead, client-go wraps the last known state
in a cache.DeletedFinalStateUnknown tombstone and passes that to the DeleteFunc
handler.

Every DeleteFunc in pkg/controller/chi/controller.go performs an unchecked bare
type-assertion on the raw obj:

// before — panics if obj is a tombstone
chi := obj.(*api.ClickHouseInstallation)

`E0419 12:43:03.457392       1 announcer.go:274] err: clickhousekeeperinstallations.clickhouse-keeper.altinity.com "clickhouse-keeper" is forbidden: User "system:serviceaccount:instana-clickhouse:ibm-clickhouse-operator" cannot update resource "clickhousekeeperinstallations/status" in API group "clickhouse-keeper.altinity.com" in the namespace "instana-clickhouse"
E0419 12:43:03.459991       1 announcer.go:274] err: clickhousekeeperinstallations.clickhouse-keeper.altinity.com "clickhouse-keeper" is forbidden: User "system:serviceaccount:instana-clickhouse:ibm-clickhouse-operator" cannot update resource "clickhousekeeperinstallations/status" in API group "clickhouse-keeper.altinity.com" in the namespace "instana-clickhouse"
E0419 12:43:03.462136       1 announcer.go:274] err: clickhousekeeperinstallations.clickhouse-keeper.altinity.com "clickhouse-keeper" is forbidden: User "system:serviceaccount:instana-clickhouse:ibm-clickhouse-operator" cannot update resource "clickhousekeeperinstallations/status" in API group "clickhouse-keeper.altinity.com" in the namespace "instana-clickhouse"
E0419 12:43:03.464653       1 announcer.go:274] err: clickhousekeeperinstallations.clickhouse-keeper.altinity.com "clickhouse-keeper" is forbidden: User "system:serviceaccount:instana-clickhouse:ibm-clickhouse-operator" cannot update resource "clickhousekeeperinstallations/status" in API group "clickhouse-keeper.altinity.com" in the namespace "instana-clickhouse"
E0419 12:43:03.468177       1 announcer.go:274] err: clickhousekeeperinstallations.clickhouse-keeper.altinity.com "clickhouse-keeper" is forbidden: User "system:serviceaccount:instana-clickhouse:ibm-clickhouse-operator" cannot update resource "clickhousekeeperinstallations/status" in API group "clickhouse-keeper.altinity.com" in the namespace "instana-clickhouse": RBAC: [clusterrole.rbac.authorization.k8s.io "system:public-info-viewer" not found, clusterrole.rbac.authorization.k8s.io "system:basic-user" not found, clusterrole.rbac.authorization.k8s.io "ibm-clickhouse-operator" not found, clusterrole.rbac.authorization.k8s.io "system:service-account-issuer-discovery" not found, clusterrole.rbac.authorization.k8s.io "system:discovery" not found]
E0419 12:43:08.068236       1 runtime.go:79] Observed a panic: &runtime.TypeAssertionError{_interface:(*abi.Type)(0x1d5d0c0), concrete:(*abi.Type)(0x1ea3ce0), asserted:(*abi.Type)(0x2084360), missingMethod:""} (interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *v1.Pod)
goroutine 146 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x1dcbb60, 0xc0016c08d0})
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:75 +0x90
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc000340380?})
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:49 +0x65
panic({0x1dcbb60?, 0xc0016c08d0?})
	/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.8.linux-amd64/src/runtime/panic.go:783 +0x132
github.com/altinity/clickhouse-operator/pkg/controller/chi.(*Controller).addEventHandlersPod.func3({0x1ea3ce0?, 0xc0001a5f00?})
	/clickhouse-operator/pkg/controller/chi/controller.go:431 +0x234
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnDelete(...)
	/clickhouse-operator/vendor/k8s.io/client-go/tools/cache/controller.go:257
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/clickhouse-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:978 +0x16f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0000a7008?)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:226 +0x33
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0008cff70, {0x239f5c0, 0xc000342060}, 0x1, 0xc00018ae00)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:227 +0xaf
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000187f70, 0x3b9aca00, 0x0, 0x1, 0xc00018ae00)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:204 +0x7f
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0002683f0)
	/clickhouse-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:967 +0x5a
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:72 +0x4c
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 125
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:70 +0x73
panic: interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *v1.Pod [recovered, repanicked]

goroutine 146 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc000340380?})
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:56 +0xc7
panic({0x1dcbb60?, 0xc0016c08d0?})
	/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.8.linux-amd64/src/runtime/panic.go:783 +0x132
github.com/altinity/clickhouse-operator/pkg/controller/chi.(*Controller).addEventHandlersPod.func3({0x1ea3ce0?, 0xc0001a5f00?})
	/clickhouse-operator/pkg/controller/chi/controller.go:431 +0x234
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnDelete(...)
	/clickhouse-operator/vendor/k8s.io/client-go/tools/cache/controller.go:257
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/clickhouse-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:978 +0x16f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0000a7008?)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:226 +0x33
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0008cff70, {0x239f5c0, 0xc000342060}, 0x1, 0xc00018ae00)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:227 +0xaf
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000187f70, 0x3b9aca00, 0x0, 0x1, 0xc00018ae00)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:204 +0x7f
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0002683f0)
	/clickhouse-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:967 +0x5a
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:72 +0x4c
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 125
	/clickhouse-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:70 +0x73
`