added a 'report missing' feature

README.md

@@ -6,6 +6,7 @@ Prometheus exporter that mines /proc to report on selected processes.
 [![Release](https://img.shields.io/github/release/ncabatoff/process-exporter.svg?style=flat-square")][release]
 [![Powered By: GoReleaser](https://img.shields.io/badge/powered%20by-goreleaser-green.svg?branch=master)](https://github.com/goreleaser)
 [![CircleCI](https://circleci.com/gh/ncabatoff/process-exporter.svg?style=shield)](https://circleci.com/gh/ncabatoff/process-exporter)
+
 Some apps are impractical to instrument directly, either because you
 don't control the code or they're written in a language that isn't easy to
 instrument with Prometheus.  We must instead resort to mining /proc.
@@ -50,6 +51,11 @@ it's assumed its proper name.
 -procnames is intended as a quick alternative to using a config file.  Details
 in the following section.
 
+-report.missing will report any processes that are not running the process-exporter
+is started as having "num_procs" of zero. You need to use a config file for this to
+work. It will use the "name" field first and if this is not defined it extracts the
+process names from the "comm" and "exe" arrays.
+
 ## Configuration and group naming
 
 To select and group the processes to monitor, either provide command-line

cmd/process-exporter/main.go

@@ -303,6 +303,7 @@ func main() {
 			"log debugging information to stdout")
 		showVersion = flag.Bool("version", false,
 			"print version information and exit")
+		reportMissing = flag.Bool("report.missing", false, "report a stopped process as having zero processes running when process exporter is first started")
 	)
 	flag.Parse()
 
@@ -317,13 +318,15 @@ func main() {
 	}
 
 	var matchnamer common.MatchNamer
+	var processNames *[]string
 
 	if *configPath != "" {
 		if *nameMapping != "" || *procNames != "" {
 			log.Fatalf("-config.path cannot be used with -namemapping or -procnames")
 		}
 
-		cfg, err := config.ReadFile(*configPath, *debug)
+		cfg, pNames, err := config.ReadFile(*configPath, *debug)
+		processNames = pNames
 		if err != nil {
 			log.Fatalf("error reading config file %q: %v", *configPath, err)
 		}
@@ -355,7 +358,7 @@ func main() {
 		matchnamer = namemapper
 	}
 
-	pc, err := NewProcessCollector(*procfsPath, *children, *threads, matchnamer, *recheck, *debug)
+	pc, err := NewProcessCollector(*procfsPath, *children, *threads, matchnamer, *recheck, *debug, *processNames, *reportMissing)
 	if err != nil {
 		log.Fatalf("Error initializing: %v", err)
 	}
@@ -404,6 +407,8 @@ type (
 		scrapeProcReadErrors int
 		scrapePartialErrors  int
 		debug                bool
+		processNames         []string
+		reportMissing        bool
 	}
 )
 
@@ -414,17 +419,21 @@ func NewProcessCollector(
 	n common.MatchNamer,
 	recheck bool,
 	debug bool,
+	processNames []string,
+	reportMissing bool,
 ) (*NamedProcessCollector, error) {
 	fs, err := proc.NewFS(procfsPath, debug)
 	if err != nil {
 		return nil, err
 	}
 	p := &NamedProcessCollector{
-		scrapeChan: make(chan scrapeRequest),
-		Grouper:    proc.NewGrouper(n, children, threads, recheck, debug),
-		source:     fs,
-		threads:    threads,
-		debug:      debug,
+		scrapeChan:    make(chan scrapeRequest),
+		Grouper:       proc.NewGrouper(n, children, threads, recheck, debug),
+		source:        fs,
+		threads:       threads,
+		debug:         debug,
+		processNames:  processNames,
+		reportMissing: reportMissing,
 	}
 
 	colErrs, _, err := p.Update(p.source.AllProcs())
@@ -491,6 +500,17 @@ func (p *NamedProcessCollector) scrape(ch chan<- prometheus.Metric) {
 		p.scrapeErrors++
 		log.Printf("error reading procs: %v", err)
 	} else {
+		if p.reportMissing {
+			// loop over all process names, if process does not have process running (in groups) then report num_procs as zero
+			for _, pName := range p.processNames {
+				_, present := groups[pName]
+				if !present {
+					ch <- prometheus.MustNewConstMetric(numprocsDesc,
+						prometheus.GaugeValue, float64(0), pName)
+				}
+			}
+		}
+
 		for gname, gcounts := range groups {
 			ch <- prometheus.MustNewConstMetric(numprocsDesc,
 				prometheus.GaugeValue, float64(gcounts.Procs), gname)

config/config.go

@@ -174,11 +174,77 @@ func (m andMatcher) Match(nacl common.ProcAttributes) bool {
 	return true
 }
 
+// getProcessNames extracts teh anmes of the processes from the given procname
+func getProcessNames(procname interface{}) []string {
+	nm, ok := procname.(map[interface{}]interface{})
+	if !ok {
+		return nil
+	}
+
+	var names []string
+	//check for 'name' field. If contains name field other fields are not extracted
+	for k, v := range nm {
+		key, ok := k.(string)
+		if !ok {
+			return nil
+		}
+		if key == "name" {
+			value, ok := v.(string)
+			if !ok {
+				return nil
+			}
+			names = append(names, value)
+			return names
+		}
+	}
+
+	for k, v := range nm {
+		key, ok := k.(string)
+		if !ok {
+			return nil
+		}
+
+		if key == "comm" {
+			// "comm" block in config file - extract values as is from array
+			values, ok := v.([]interface{})
+			if !ok {
+				return nil
+			}
+			for _, rawValue := range values {
+				value, ok := rawValue.(string)
+				if !ok {
+					return nil
+				}
+				names = append(names, value)
+			}
+		} else if key == "exe" {
+			// "exe" block in config file - extracts names from array
+			exes, ok := v.([]interface{})
+			if !ok {
+				return nil
+			}
+			for _, rawValue := range exes {
+				value, ok := rawValue.(string)
+				if !ok {
+					return nil
+				}
+				// check for forward slash - need to extract filename if "/" is present
+				if strings.Contains(value, "/") {
+					names = append(names, filepath.Base(value))
+				} else {
+					names = append(names, value)
+				}
+			}
+		}
+	}
+	return names
+}
+
 // ReadRecipesFile opens the named file and extracts recipes from it.
-func ReadFile(cfgpath string, debug bool) (*Config, error) {
+func ReadFile(cfgpath string, debug bool) (*Config, *[]string, error) {
 	content, err := ioutil.ReadFile(cfgpath)
 	if err != nil {
-		return nil, fmt.Errorf("error reading config file %q: %v", cfgpath, err)
+		return nil, nil, fmt.Errorf("error reading config file %q: %v", cfgpath, err)
 	}
 	if debug {
 		log.Printf("Config file %q contents:\n%s", cfgpath, content)
@@ -187,32 +253,37 @@ func ReadFile(cfgpath string, debug bool) (*Config, error) {
 }
 
 // GetConfig extracts Config from content by parsing it as YAML.
-func GetConfig(content string, debug bool) (*Config, error) {
+func GetConfig(content string, debug bool) (*Config, *[]string, error) {
 	var yamldata map[string]interface{}
 
 	err := yaml.Unmarshal([]byte(content), &yamldata)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	yamlProcnames, ok := yamldata["process_names"]
 	if !ok {
-		return nil, fmt.Errorf("error parsing YAML config: no top-level 'process_names' key")
+		return nil, nil, fmt.Errorf("error parsing YAML config: no top-level 'process_names' key")
 	}
 	procnames, ok := yamlProcnames.([]interface{})
 	if !ok {
-		return nil, fmt.Errorf("error parsing YAML config: 'process_names' is not a list")
+		return nil, nil, fmt.Errorf("error parsing YAML config: 'process_names' is not a list")
 	}
 
 	var cfg Config
+	var processNames []string
 	for i, procname := range procnames {
 		mn, err := getMatchNamer(procname)
 		if err != nil {
-			return nil, fmt.Errorf("unable to parse process_name entry %d: %v", i, err)
+			return nil, nil, fmt.Errorf("unable to parse process_name entry %d: %v", i, err)
 		}
 		cfg.MatchNamers.matchers = append(cfg.MatchNamers.matchers, mn)
+
+		// get names of all processes
+		pNames := getProcessNames(procname)
+		processNames = append(processNames, pNames...)
 	}
 
-	return &cfg, nil
+	return &cfg, &processNames, nil
 }
 
 func getMatchNamer(yamlmn interface{}) (common.MatchNamer, error) {

config/config_test.go

@@ -16,7 +16,7 @@ process_names:
   - exe: 
     - /bin/ksh
 `
-	cfg, err := GetConfig(yml, false)
+	cfg, _, err := GetConfig(yml, false)
 	c.Assert(err, IsNil)
 	c.Check(cfg.MatchNamers.matchers, HasLen, 3)
 
@@ -61,7 +61,7 @@ process_names:
     - prometheus
     name: "{{.ExeFull}}"
 `
-	cfg, err := GetConfig(yml, false)
+	cfg, _, err := GetConfig(yml, false)
 	c.Assert(err, IsNil)
 	c.Check(cfg.MatchNamers.matchers, HasLen, 2)
 
@@ -75,3 +75,24 @@ process_names:
 	c.Check(found, Equals, true)
 	c.Check(name, Equals, "/usr/local/bin/prometheus")
 }
+
+func (s MySuite) TestReportMissingFeature(c *C) {
+	yml := `
+process_names:
+  - comm:
+    - prometheus
+    - grafana
+  - exe:
+    - postmaster
+    - anotherExe
+    cmdline:
+    - "-a -b --verbose"
+  - exe:
+    - yetAnotherExe
+    name: "named_exe"
+  `
+
+	_, processNames, err := GetConfig(yml, false)
+	c.Assert(err, IsNil)
+	c.Check(*processNames, DeepEquals, []string{"prometheus", "grafana", "postmaster", "anotherExe", "named_exe"})
+}