feat: fix issues with metrics collection and enhanced configuration

Makefile

@@ -27,3 +27,6 @@ release: goreleaser
 .PHONY: goreleaser
 goreleaser:
 	@go get github.com/goreleaser/goreleaser && go install github.com/goreleaser/goreleaser
+
+docker:
+	docker build -t foomo/pagespeed_exporter:latest .

README.md

@@ -47,7 +47,34 @@ Instructions how to create a key for pagespeed can be found [here](https://devel
 $ pagespeed_exporter -api-key {KEY} -targets https://google.com,https://prometheus.io -listener :80
 ```
 
-### Exporter configuration
+### Exporter Target Specification
+
+Targets can be configured in either plaintext 
+
+```
+https://github.com/foomo/pagespeed_exporter
+https://mysite.com/test?test=true
+```
+
+Or via JSON which adds additional parameters
+
+```
+// URL can't be invalid
+// Strategy can only be mobile/desktop
+// If strategy is not specified, both desktop & mobile will be used
+// Parameters are passed down to google pagespeed api
+
+{"url":"https://github.com/foomo/pagespeed_exporter","campaign":"test","locale":"en","source":"source"}
+
+{"url":"https://mysite.com/test?test=true","strategy":"mobile"}
+
+```
+
+Configuration specification in JSON and plain is supported both in command line & prometheus configuration 
+
+### Exporter configuration 
+
+Configuration of targets can be done via docker and via prometheus
 
 | Flag      | Variable           | Description                                   | Default | Required |
 |-----------|--------------------|-----------------------------------------------|---------|----------|
@@ -58,6 +85,37 @@ $ pagespeed_exporter -api-key {KEY} -targets https://google.com,https://promethe
 | -parallel | PAGESPEED_PARALLEL | sets the execution of targets to be parallel  | false   | False    |
 
 Note: google api key is required only if scraping more than 2 targets/second
+
+Note: exporter can be run without targets, and later targets provided via prometheus
+
+### Exporter Target Configuration (VIA PROMETHEUS)
+
+Example configuration with simple and complex values
+
+(Examples can ve found in the example folder)
+
+```yaml
+
+  - job_name: pagespeed_exporter_probe
+      metrics_path: /probe
+      # Re-Label configurations so that we can use them
+      # to configure the pagespeed exporter
+      relabel_configs:
+        - source_labels: [__address__]
+          target_label: __param_target
+        - source_labels: [__param_target]
+          target_label: instance
+        - target_label: __address__
+          replacement: "pagespeed_exporter:9271"
+      static_configs:
+        - targets:
+            - 'https://example.com/' # Example PLAIN
+            - '{"url":"https://example.com/","campaign":"test","locale":"en","source":"source"}'  
+            - '{"url":"https://example.com/mobileonly","strategy":"mobile"}'                    
+
+```
+
+
 ### Docker
 
 ```sh

collector/scrape.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"github.com/gammazero/workerpool"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
 	"google.golang.org/api/googleapi/transport"
 	"google.golang.org/api/option"
@@ -60,10 +60,10 @@ func (pss *pagespeedScrapeService) Scrape(parallel bool, requests []ScrapeReques
 		request := req
 		wp.Submit(func() {
 
-			scrape, err := pss.scrape(req)
+			scrape, err := pss.scrape(request)
 			if err != nil {
-				logrus.WithError(err).
-					WithFields(logrus.Fields{
+				log.WithError(err).
+					WithFields(log.Fields{
 						"target":   request.Url,
 						"strategy": request.Strategy,
 					}).Warn("target scraping returned an error")
@@ -86,7 +86,6 @@ func (pss *pagespeedScrapeService) Scrape(parallel bool, requests []ScrapeReques
 }
 
 func (pss pagespeedScrapeService) scrape(request ScrapeRequest) (scrape *ScrapeResult, err error) {
-
 	service, err := pagespeedonline.NewService(context.Background(), option.WithHTTPClient(pss.scrapeClient))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize pagespeed service")

example/docker-compose.yml

@@ -8,6 +8,8 @@ services:
   pagespeed_exporter:
     image: foomo/pagespeed_exporter:latest
     restart: unless-stopped
+    ports:
+      - "9271:9271"
     command:
       - "-t=https://www.google.com"
       - "-t=https://www.google.com/webhp"

example/prometheus/config.yml

@@ -1,5 +1,5 @@
 global:
-  scrape_interval: 1h # By default, scrape targets every 15 seconds.
+  scrape_interval: 30s # By default, scrape targets every 15 seconds.
   scrape_timeout: 30s
 
 scrape_configs:
@@ -10,17 +10,16 @@ scrape_configs:
           - 'pagespeed_exporter:9271'
 
   - job_name: pagespeed_exporter_probe
-      metrics_path: /probe
-      # Re-Label configurations so that we can use them
-      # to configure the pagespeed exporter
-      relabel_configs:
-        - source_labels: [__address__]
-          target_label: __param_target
-        - source_labels: [__param_target]
-          target_label: instance
-        - target_label: __address__
-          replacement: "pagespeed_exporter:9271"
-      static_configs:
-        - targets:
-            - 'https://example.com/' # Example PLAIN
-            - '{"url":"https://example.com/","campaign":"test","locale":"en","source":"source"}'  # Example Complex
+    metrics_path: /probe
+    # Re-Label configurations so that we can use them
+    # to configure the pagespeed exporter
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - target_label: __address__
+        replacement: "pagespeed_exporter:9271"
+    static_configs:
+      - targets:
+          - 'https://github.com/'
+          - '{"url":"https://github.com/foomo","campaign":"test","locale":"en","source":"source"}'
+          - '{"url":"https://github.com/foomo/pagespeed_exporter","strategy":"mobile"}'

handler/probe.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"fmt"
 	"github.com/foomo/pagespeed_exporter/collector"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -37,6 +36,11 @@ func (ph httpProbeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/plain")
 
 	targets := r.URL.Query()["target"]
+
+	for _, target := range targets {
+		log.WithField("target", target).Info("probe requested for target")
+	}
+
 	requests := collector.CalculateScrapeRequests(targets...)
 	if len(requests) == 0 {
 		http.Error(w, "Probe requires at least one target", http.StatusBadRequest)
@@ -83,12 +87,15 @@ func errResponse(w http.ResponseWriter, message string, err error) {
 
 func getScrapeTimeout(r *http.Request) (timeout time.Duration, err error) {
 	// If a timeout is configured via the Prometheus header, add it to the request.
-	fmt.Println("HAEDER:", r.Header.Get(PrometheusTimeoutHeader))
 	if v := r.Header.Get(PrometheusTimeoutHeader); v != "" {
 		timeoutSeconds, err := strconv.ParseFloat(v, 64)
 		if err != nil {
 			return 0, errors.Wrap(err, "could not parse timeout")
 		}
+		if timeoutSeconds == 0 {
+			return DefaultTimeoutDuration, nil
+		}
+
 		return time.Duration(timeoutSeconds*float64(time.Second)) - DefaultTimeOffset, nil
 	}