Merge pull request #8787 from cd1989/core-hunging

Fix core hung when stop problem

src/core/api/registry_test.go

@@ -35,7 +35,7 @@ type RegistrySuite struct {
 
 func (suite *RegistrySuite) SetupSuite() {
 	assert := assert.New(suite.T())
-	assert.Nil(replication.Init(make(chan struct{})))
+	assert.Nil(replication.Init(make(chan struct{}), make(chan struct{})))
 
 	suite.testAPI = newHarborAPI()
 	code, err := suite.testAPI.RegistryCreate(*admin, testRegistry)

src/core/main.go

@@ -17,6 +17,12 @@ package main
 import (
 	"encoding/gob"
 	"fmt"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
 	"github.com/astaxie/beego"
 	_ "github.com/astaxie/beego/session/redis"
 	"github.com/goharbor/harbor/src/common/dao"
@@ -30,10 +36,6 @@ import (
 	_ "github.com/goharbor/harbor/src/core/auth/db"
 	_ "github.com/goharbor/harbor/src/core/auth/ldap"
 	_ "github.com/goharbor/harbor/src/core/auth/uaa"
-	"os"
-	"os/signal"
-	"strconv"
-	"syscall"
 
 	quota "github.com/goharbor/harbor/src/core/api/quota"
 	_ "github.com/goharbor/harbor/src/core/api/quota/chart"
@@ -138,17 +140,24 @@ func quotaSync() error {
 	return nil
 }
 
-func gracefulShutdown(closing chan struct{}) {
+func gracefulShutdown(closing, done chan struct{}) {
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
 	log.Infof("capture system signal %s, to close \"closing\" channel", <-signals)
 	close(closing)
+	select {
+	case <-done:
+		log.Infof("Goroutines exited normally")
+	case <-time.After(time.Second * 3):
+		log.Infof("Timeout waiting goroutines to exit")
+	}
+	os.Exit(0)
 }
 
 func main() {
 	beego.BConfig.WebConfig.Session.SessionOn = true
 	beego.BConfig.WebConfig.Session.SessionName = "sid"
-	// TODO
+
 	redisURL := os.Getenv("_REDIS_URL")
 	if len(redisURL) > 0 {
 		gob.Register(models.User{})
@@ -203,8 +212,9 @@ func main() {
 	}
 
 	closing := make(chan struct{})
-	go gracefulShutdown(closing)
-	if err := replication.Init(closing); err != nil {
+	done := make(chan struct{})
+	go gracefulShutdown(closing, done)
+	if err := replication.Init(closing, done); err != nil {
 		log.Fatalf("failed to init for replication: %v", err)
 	}
 

src/replication/registry/healthcheck.go

@@ -29,17 +29,19 @@ const MinInterval = time.Minute * 5
 type HealthChecker struct {
 	interval time.Duration
 	closing  chan struct{}
+	done     chan struct{}
 	manager  Manager
 }
 
 // NewHealthChecker creates a new health checker
 // - interval specifies the time interval to perform health check for registries
 // - closing is a channel to stop the health checker
-func NewHealthChecker(interval time.Duration, closing chan struct{}) *HealthChecker {
+func NewHealthChecker(interval time.Duration, closing, done chan struct{}) *HealthChecker {
 	return &HealthChecker{
 		interval: interval,
 		manager:  NewDefaultManager(),
 		closing:  closing,
+		done:     done,
 	}
 }
 
@@ -66,6 +68,8 @@ func (c *HealthChecker) Run() {
 			log.Debug("Health Check succeeded")
 		case <-c.closing:
 			log.Info("Stop health checker")
+			// No cleanup works to do, signal done directly
+			close(c.done)
 			return
 		}
 	}

src/replication/replication.go

@@ -59,7 +59,7 @@ var (
 )
 
 // Init the global variables and configurations
-func Init(closing chan struct{}) error {
+func Init(closing, done chan struct{}) error {
 	// init config
 	secretKey, err := cfg.SecretKey()
 	if err != nil {
@@ -86,6 +86,6 @@ func Init(closing chan struct{}) error {
 	log.Debug("the replication initialization completed")
 
 	// Start health checker for registries
-	go registry.NewHealthChecker(time.Minute*5, closing).Run()
+	go registry.NewHealthChecker(time.Minute*5, closing, done).Run()
 	return nil
 }

src/replication/replication_test.go

@@ -34,7 +34,7 @@ func TestInit(t *testing.T) {
 	require.Nil(t, err)
 
 	config.InitWithSettings(nil)
-	err = Init(make(chan struct{}))
+	err = Init(make(chan struct{}), make(chan struct{}))
 	require.Nil(t, err)
 	assert.NotNil(t, PolicyCtl)
 	assert.NotNil(t, RegistryMgr)