Cherry-pick #18916 to 7.8: Windows: fix service termination

CHANGELOG.next.asciidoc

@@ -117,6 +117,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Autodiscover] Check if runner is already running before starting again. {pull}18564[18564]
 - Fix `keystore add` hanging under Windows. {issue}18649[18649] {pull}18654[18654]
 - Fix regression in `add_kubernetes_metadata`, so configured `indexers` and `matchers` are used if defaults are not disabled. {issue}18481[18481] {pull}18818[18818]
+- Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916]
 
 *Auditbeat*
 

libbeat/cmd/instance/beat.go

@@ -390,6 +390,12 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 		return err
 	}
 
+	// Windows: Mark service as stopped.
+	// After this is run, a Beat service is considered by the OS to be stopped
+	// and another instance of the process can be started.
+	// This must be the first deferred cleanup task (last to execute).
+	defer svc.NotifyTermination()
+
 	// Try to acquire exclusive lock on data path to prevent another beat instance
 	// sharing same data path.
 	bl := newLocker(b)

libbeat/service/service.go

@@ -67,6 +67,11 @@ func HandleSignals(stopFunction func(), cancel context.CancelFunc) {
 	})
 }
 
+// NotifyTermination tells the OS that the service is stopped.
+func NotifyTermination() {
+	notifyWindowsServiceStopped()
+}
+
 // cmdline flags
 var memprofile, cpuprofile, httpprof *string
 var cpuOut *os.File

libbeat/service/service_unix.go

@@ -22,3 +22,6 @@ package service
 // ProcessWindowsControlEvents is not used on non-windows platforms.
 func ProcessWindowsControlEvents(stopCallback func()) {
 }
+
+func notifyWindowsServiceStopped() {
+}

libbeat/service/service_windows.go

@@ -28,7 +28,15 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
-type beatService struct{}
+type beatService struct {
+	stopCallback func()
+	done         chan struct{}
+}
+
+var serviceInstance = &beatService{
+	stopCallback: nil,
+	done:         make(chan struct{}, 0),
+}
 
 // Execute runs the beat service with the arguments and manages changes that
 // occur in the environment or runtime that may affect the beat.
@@ -52,9 +60,22 @@ loop:
 		}
 	}
 	changes <- svc.Status{State: svc.StopPending}
+	m.stopCallback()
+	// Block until notifyWindowsServiceStopped below is called. This is required
+	// as the windows/svc package will transition the service to STOPPED state
+	// once this function returns.
+	<-m.done
 	return
 }
 
+func (m *beatService) stop() {
+	close(m.done)
+}
+
+func notifyWindowsServiceStopped() {
+	serviceInstance.stop()
+}
+
 // couldNotConnect is the errno for ERROR_FAILED_SERVICE_CONTROLLER_CONNECT.
 const couldNotConnect syscall.Errno = 1063
 
@@ -76,10 +97,10 @@ func ProcessWindowsControlEvents(stopCallback func()) {
 		run = debug.Run
 	}
 
-	err = run(os.Args[0], &beatService{})
+	serviceInstance.stopCallback = stopCallback
+	err = run(os.Args[0], serviceInstance)
 
 	if err == nil {
-		stopCallback()
 		return
 	}