Merge pull request #3 from abstractmj/fea-clb-portpool

fix: fix deployment update resource hang forever problem in zookeeper store
TencentBlueKing · Jul 6, 2021 · 2f18cc4 · 2f18cc4
2 parents 7e58534 + 185d777
commit 2f18cc4
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 39 deletions.
diff --git a/bcs-mesos/bcs-scheduler/src/manager/sched/scheduler/transaction.go b/bcs-mesos/bcs-scheduler/src/manager/sched/scheduler/transaction.go
@@ -141,17 +141,31 @@ func (s *Scheduler) updateTransactionDeployment(transaction *types.Transaction)
 		ns := transaction.Namespace
 		currDeployment, err := s.store.FetchDeployment(ns, name)
 		if err != nil {
+			if errors.Is(err, store.ErrNoFound) {
+				blog.Infof("transaction %s finished, deployment(%s.%s) not found, no need update deployment",
+					transaction.TransactionID, ns, name)
+				return nil
+			}
 			return fmt.Errorf("get deployment(%s.%s) failed when finish transaction, err %s",
 				ns, name, err.Error())
 		}
 		appName := currDeployment.Application.ApplicationName
 		currApp, err := s.store.FetchApplication(ns, appName)
 		if err != nil {
-			return fmt.Errorf("get application(%s.%s) failed when UpdateDeploymentResource, err %s",
-				ns, appName, err.Error())
-		}
-		if currApp.Message != types.APP_STATUS_RUNNING_STR {
-			currDeployment.Message = currApp.Message
+			if !errors.Is(err, store.ErrNoFound) {
+				return fmt.Errorf("transaction %s get application(%s.%s) failed when UpdateDeploymentResource, err %s",
+					transaction.TransactionID, ns, appName, err.Error())
+
+			}
+			blog.Infof("transaction %s finished, application(%s.%s) of deployment(%s.%s) not found",
+				transaction.TransactionID, ns, appName, ns, name)
+			currDeployment.Message = fmt.Sprintf(
+				"transaction %s finished, application(%s.%s) of deployment(%s.%s) not found",
+				transaction.TransactionID, ns, appName, ns, name)
+		} else {
+			if currApp.Message != types.APP_STATUS_RUNNING_STR {
+				currDeployment.Message = currApp.Message
+			}
 		}
 		currDeployment.Status = types.DEPLOYMENT_STATUS_RUNNING
 		if err := s.store.SaveDeployment(currDeployment); err != nil {

diff --git a/bcs-mesos/bcs-scheduler/src/manager/store/zk/agent.go b/bcs-mesos/bcs-scheduler/src/manager/store/zk/agent.go
@@ -19,7 +19,6 @@ import (
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	commtypes "github.com/Tencent/bk-bcs/bcs-common/common/types"
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/scheduler/schetypes"
-	schStore "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/manager/store"
 
 	"github.com/samuel/go-zookeeper/zk"
 )
@@ -56,9 +55,6 @@ func (store *managerStore) FetchAgent(Key string) (*types.Agent, error) {
 
 	data, err := store.Db.Fetch(path)
 	if err != nil {
-		if err == zk.ErrNoNode {
-			return nil, schStore.ErrNoFound
-		}
 		return nil, err
 	}
 
@@ -116,12 +112,6 @@ func (store *managerStore) FetchAgentSetting(InnerIP string) (*commtypes.BcsClus
 	path := getAgentSettingRootPath() + "/" + InnerIP
 
 	data, err := store.Db.Fetch(path)
-
-	if err == zk.ErrNoNode {
-		blog.V(3).Infof("agentSetting(%s) not exist", path)
-		return nil, schStore.ErrNoFound
-	}
-
 	if err != nil {
 		return nil, err
 	}
@@ -202,12 +192,6 @@ func (store *managerStore) FetchAgentSchedInfo(HostName string) (*types.AgentSch
 	path := getAgentSchedInfoRootPath() + "/" + HostName
 
 	data, err := store.Db.Fetch(path)
-
-	if err == zk.ErrNoNode {
-		blog.V(3).Infof("agentSchedInfo(%s) not exist", path)
-		return nil, schStore.ErrNoFound
-	}
-
 	if err != nil {
 		return nil, err
 	}

diff --git a/bcs-mesos/bcs-scheduler/src/manager/store/zk/application.go b/bcs-mesos/bcs-scheduler/src/manager/store/zk/application.go
@@ -19,9 +19,6 @@ import (
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/scheduler/schetypes"
-	schStore "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/manager/store"
-
-	"github.com/samuel/go-zookeeper/zk"
 )
 
 var appLocks map[string]*sync.Mutex
@@ -123,9 +120,6 @@ func (store *managerStore) FetchApplication(runAs, appID string) (*types.Applica
 
 	data, err := store.Db.Fetch(path)
 	if err != nil {
-		if err == zk.ErrNoNode {
-			return nil, schStore.ErrNoFound
-		}
 		return nil, err
 	}
 

diff --git a/bcs-mesos/bcs-scheduler/src/manager/store/zk/dbzk.go b/bcs-mesos/bcs-scheduler/src/manager/store/zk/dbzk.go
@@ -18,6 +18,9 @@ import (
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/zkclient"
 	"github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/manager/store"
+	schStore "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/manager/store"
+
+	gozk "github.com/samuel/go-zookeeper/zk"
 )
 
 //dbZk is a struct of the zookeeper client
@@ -63,6 +66,9 @@ func (zk *dbZk) Fetch(path string) ([]byte, error) {
 
 	data, err := zk.ZkCli.Get(path)
 	if err != nil {
+		if err == gozk.ErrNoNode {
+			err = schStore.ErrNoFound
+		}
 		failed = true
 	}
 

diff --git a/bcs-mesos/bcs-scheduler/src/manager/store/zk/deployment.go b/bcs-mesos/bcs-scheduler/src/manager/store/zk/deployment.go
@@ -19,9 +19,6 @@ import (
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/scheduler/schetypes"
-	schStore "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/manager/store"
-
-	"github.com/samuel/go-zookeeper/zk"
 )
 
 var deploymentLocks map[string]*sync.Mutex
@@ -87,9 +84,6 @@ func (store *managerStore) FetchDeployment(ns, name string) (*types.Deployment,
 	path := getDeploymentRootPath() + ns + "/" + name
 	data, err := store.Db.Fetch(path)
 	if err != nil {
-		if err == zk.ErrNoNode {
-			return nil, schStore.ErrNoFound
-		}
 		return nil, err
 	}
 	deployment := &types.Deployment{}

diff --git a/bcs-mesos/bcs-scheduler/src/manager/store/zk/task.go b/bcs-mesos/bcs-scheduler/src/manager/store/zk/task.go
@@ -19,9 +19,6 @@ import (
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/scheduler/schetypes"
-	schStore "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/manager/store"
-
-	"github.com/samuel/go-zookeeper/zk"
 )
 
 func getTaskRootPath() string {
@@ -141,9 +138,6 @@ func (store *managerStore) FetchTask(taskId string) (*types.Task, error) {
 
 	data, err := store.Db.Fetch(path)
 	if err != nil {
-		if err == zk.ErrNoNode {
-			return nil, schStore.ErrNoFound
-		}
 		return nil, err
 	}