Bug: try dlock before start retirement job (#1193)

datasource/etcd/dlock.go

@@ -18,12 +18,12 @@
 package etcd
 
 import (
+	"fmt"
 	"sync"
 
-	"github.com/go-chassis/openlog"
-	"github.com/little-cui/etcdadpt"
-
 	"github.com/apache/servicecomb-service-center/datasource/dlock"
+	"github.com/apache/servicecomb-service-center/pkg/log"
+	"github.com/little-cui/etcdadpt"
 )
 
 func init() {
@@ -60,7 +60,7 @@ func (d *DB) Renew(key string) error {
 	if lock, ok := d.lockMap.Load(key); ok {
 		err := lock.(*etcdadpt.DLock).Refresh()
 		if err != nil {
-			openlog.Error("fail to renew key")
+			log.Error(fmt.Sprintf("fail to renew key %s", key), err)
 			d.lockMap.Delete(key)
 		}
 		return err
@@ -81,7 +81,7 @@ func (d *DB) Unlock(key string) error {
 	if lock, ok := d.lockMap.Load(key); ok {
 		err := lock.(*etcdadpt.DLock).Unlock()
 		if err != nil {
-			openlog.Error("fail to unlock")
+			log.Error(fmt.Sprintf("fail to unlock %s", key), err)
 		}
 		d.lockMap.Delete(key)
 		return err

server/job/disco/schema.go

@@ -24,27 +24,41 @@ import (
 	"github.com/apache/servicecomb-service-center/pkg/log"
 	"github.com/apache/servicecomb-service-center/server/config"
 	discosvc "github.com/apache/servicecomb-service-center/server/service/disco"
+	"github.com/apache/servicecomb-service-center/server/service/dlock"
 	"github.com/robfig/cron/v3"
 )
 
 const (
 	defaultRetireSchemaCron = "0 2 * * *"
+	retireSchemaLockTTL     = 60
+	retireSchemaLockKey     = "retire-schema-job"
 )
 
 func init() {
 	cronExpr := config.GetString("registry.schema.retire.cron", defaultRetireSchemaCron)
 	log.Info(fmt.Sprintf("start retire schema job, plan is %v", cronExpr))
 	c := cron.New()
 	_, err := c.AddFunc(cronExpr, func() {
-		//TODO use DLock
-		err := discosvc.RetireSchema(context.Background())
-		if err != nil {
-			log.Error("retire schema failed", err)
-		}
+		retireSchema()
 	})
 	if err != nil {
 		log.Error("cron add func failed", err)
 		return
 	}
 	c.Start()
 }
+
+func retireSchema() {
+	err := dlock.TryLock(retireSchemaLockKey, retireSchemaLockTTL)
+	if err != nil {
+		log.Error(fmt.Sprintf("try lock %s failed", retireSchemaLockKey), err)
+		return
+	}
+	defer dlock.Unlock(retireSchemaLockKey)
+
+	log.Info("start retire schema")
+	err = discosvc.RetireSchema(context.Background())
+	if err != nil {
+		log.Error("retire schema failed", err)
+	}
+}

server/job/disco/retire.go → server/job/disco/service.go

@@ -25,12 +25,15 @@ import (
 	"github.com/apache/servicecomb-service-center/pkg/log"
 	"github.com/apache/servicecomb-service-center/server/config"
 	discosvc "github.com/apache/servicecomb-service-center/server/service/disco"
+	"github.com/apache/servicecomb-service-center/server/service/dlock"
 	"github.com/robfig/cron/v3"
 )
 
 const (
 	defaultRetireMicroserviceCron = "0 1 * * *"
 	defaultReserveVersionCount    = 3
+	retireServiceLockTTL          = 60
+	retireServiceLockKey          = "retire-service-job"
 )
 
 func init() {
@@ -50,15 +53,26 @@ func startRetireServiceJob() {
 
 	c := cron.New()
 	_, err := c.AddFunc(localPlan.Cron, func() {
-		//TODO use DLock
-		err := discosvc.RetireService(context.Background(), localPlan)
-		if err != nil {
-			log.Error("retire microservice failed", err)
-		}
+		retireService(localPlan)
 	})
 	if err != nil {
 		log.Error("cron add func failed", err)
 		return
 	}
 	c.Start()
 }
+
+func retireService(localPlan *datasource.RetirePlan) {
+	err := dlock.TryLock(retireServiceLockKey, retireServiceLockTTL)
+	if err != nil {
+		log.Error(fmt.Sprintf("try lock %s failed", retireServiceLockKey), err)
+		return
+	}
+	defer dlock.Unlock(retireServiceLockKey)
+
+	log.Info("start retire microservice")
+	err = discosvc.RetireService(context.Background(), localPlan)
+	if err != nil {
+		log.Error("retire microservice failed", err)
+	}
+}

server/service/dlock/dlock.go

@@ -19,7 +19,10 @@
 package dlock
 
 import (
+	"fmt"
+
 	"github.com/apache/servicecomb-service-center/datasource/dlock"
+	"github.com/apache/servicecomb-service-center/pkg/log"
 )
 
 func Lock(key string, ttl int64) error {
@@ -38,6 +41,9 @@ func IsHoldLock(key string) bool {
 	return dlock.Instance().IsHoldLock(key)
 }
 
-func Unlock(key string) error {
-	return dlock.Instance().Unlock(key)
+func Unlock(key string) {
+	err := dlock.Instance().Unlock(key)
+	if err != nil {
+		log.Error(fmt.Sprintf("unlock key %s failed", key), err)
+	}
 }

server/service/dlock/dlock_test.go

@@ -87,8 +87,9 @@ func TestDLock(t *testing.T) {
 		t.Run("unlock the unlock key should pass", func(t *testing.T) {
 			err := dlock.Lock("unlock", 5)
 			assert.Nil(t, err)
-			err = dlock.Unlock("unlock")
-			assert.Nil(t, err)
+			dlock.Unlock("unlock")
+			lock := dlock.IsHoldLock("unlock")
+			assert.False(t, lock)
 		})
 	})
 }