Fix so plugin will get unloaded and tasks disabled after plugin killed

control/available_plugin.go

@@ -302,13 +302,6 @@ func (a *availablePlugin) Kill(r string) error {
 // CheckHealth checks the health of a plugin and updates
 // a.failedHealthChecks
 func (a *availablePlugin) CheckHealth() {
-	if a.IsRemote() {
-		runnerLog.WithFields(log.Fields{
-			"_module": "control-aplugin",
-			"_block":  "check-health",
-		}).Debug(fmt.Sprintf("bypassing check-health on standalone plugin"))
-		return
-	}
 	go func() {
 		a.healthChan <- a.client.Ping()
 	}()

control/plugin/client/grpc.go

@@ -538,6 +538,10 @@ func (g *grpcClient) handleInStream(
 		for {
 			in, err := g.stream.Recv()
 			if err != nil {
+				g.conn.Close()
+				if strings.Contains(err.Error(), "transport is closing") {
+					errChan <- errors.New("connection broken")
+				}
 				errChan <- err
 				break
 			}

control/plugin_manager.go

@@ -545,7 +545,9 @@ func (p *pluginManager) LoadPlugin(details *pluginDetails, emitter gomit.Emitter
 			}
 
 			colClient := ap.client.(client.PluginCollectorClient)
-			defer ap.client.(client.PluginCollectorClient).Close()
+			if !ap.isRemote {
+				defer ap.client.(client.PluginCollectorClient).Close()
+			}
 
 			cfg := plugin.ConfigType{
 				ConfigDataNode: cfgNode,
@@ -653,6 +655,27 @@ func (p *pluginManager) LoadPlugin(details *pluginDetails, emitter gomit.Emitter
 			}).Error("load plugin error while adding loaded plugin to load plugins collection")
 			resultChan <- result{nil, aErr}
 		}
+		if ap.isRemote && aErr == nil {
+			// monitor standalone plugins. Unload them from the plugin catalog and metrics list
+			// when we detect they are no longer online.
+			go func() {
+				defer ap.client.(client.PluginCollectorClient).Close()
+				for {
+					time.Sleep(5 * time.Second)
+					go ap.CheckHealth()
+					if ap.failedHealthChecks > 3 {
+						p.UnloadPlugin(lPlugin)
+						return
+					}
+					if _, err := p.loadedPlugins.get(lPlugin.Key()); err != nil {
+						// prevent leaking routine when plugin is unloaded normally
+						return
+					}
+
+				}
+			}()
+
+		}
 		resultChan <- result{lPlugin, nil}
 		return
 	}()