fix: Make querycoord panick when rg metastore sync fail (#34106)

See also #34047

When `unassignNode` sync resource group with node removed failed

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/querycoordv2/meta/resource_manager.go

@@ -820,12 +820,11 @@ func (rm *ResourceManager) unassignNode(node int64) (string, error) {
 		mrg.UnassignNode(node)
 		rg := mrg.ToResourceGroup()
 		if err := rm.catalog.SaveResourceGroup(rg.GetMeta()); err != nil {
-			log.Warn("unassign node from resource group",
+			log.Fatal("unassign node from resource group",
 				zap.String("rgName", rg.GetName()),
 				zap.Int64("node", node),
 				zap.Error(err),
 			)
-			return "", merr.WrapErrResourceGroupServiceAvailable()
 		}
 
 		// Commit updates to memory.

internal/querycoordv2/meta/resource_manager_test.go

@@ -18,11 +18,13 @@ package meta
 import (
 	"testing"
 
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/rgpb"
 	"github.com/milvus-io/milvus/internal/kv"
 	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
+	"github.com/milvus-io/milvus/internal/kv/mocks"
 	"github.com/milvus-io/milvus/internal/metastore/kv/querycoord"
 	"github.com/milvus-io/milvus/internal/querycoordv2/params"
 	"github.com/milvus-io/milvus/internal/querycoordv2/session"
@@ -591,3 +593,29 @@ func (suite *ResourceManagerSuite) TestIncomingNode() {
 	suite.NoError(err)
 	suite.Len(nodes, 1)
 }
+
+func (suite *ResourceManagerSuite) TestUnassignFail() {
+	// suite.man
+	mockKV := mocks.NewMetaKv(suite.T())
+	mockKV.EXPECT().MultiSave(mock.Anything).Return(nil).Once()
+
+	store := querycoord.NewCatalog(mockKV)
+	suite.manager = NewResourceManager(store, session.NewNodeManager())
+
+	suite.manager.UpdateResourceGroups(map[string]*rgpb.ResourceGroupConfig{
+		"rg1": newResourceGroupConfig(20, 30),
+	})
+
+	suite.manager.nodeMgr.Add(session.NewNodeInfo(session.ImmutableNodeInfo{
+		NodeID:   1,
+		Address:  "localhost",
+		Hostname: "localhost",
+	}))
+	suite.manager.HandleNodeUp(1)
+
+	mockKV.EXPECT().MultiSave(mock.Anything).Return(merr.WrapErrServiceInternal("mocked")).Once()
+
+	suite.Panics(func() {
+		suite.manager.HandleNodeDown(1)
+	})
+}