feat: Add statuspage engine auto-detection

README.md

@@ -9,7 +9,10 @@ Statuspage exporter exports metrics from given statuspages as prometheus metrics
 
 ## Supported statuspage engines:
 - Statuspage.io (Widely used statuspage engine. For example by [GitHub](https://www.githubstatus.com)). You can check that statuspage is supported by this engine by checking that it has a [/api/v2/components.json](https://www.githubstatus.com/api/v2/components.json) endpoint.
-- Status.io (Widely used statuspage engine. For example by [Gitlab.com](https://status.gitlab.com).  You can check that statuspage is supported by this engine by checking footer of the page. It should contain status.io text)
+- Status.io (Widely used statuspage engine. For example by [Gitlab.com](https://status.gitlab.com). You can check that statuspage is supported by this engine by checking footer of the page. It should contain status.io text)
+
+Statuspage exporter will automatically detect, which engine used by statuspage and will use appropriate parser.
+If this statuspage is not supported by any of the engines, then statuspage exporter will show error message in the logs.
 
 ## Some popular statuspages:
 
@@ -71,11 +74,11 @@ fetch_delay: 5
 # Timeout for the http client
 client_timeout: 2
 # List of the targets to scrape
-statuspageio_pages:
+statuspages:
   - https://githubstatus.com
   - https://jira-software.status.atlassian.com
-statusio_pages:
   - https://status.gitlab.com
+retry_count: 3
 ```
 
 ## Metrics Example

main.go

@@ -11,12 +11,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/go-resty/resty/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/version"
 	"go.uber.org/zap"
 
 	"github.com/sergeyshevch/statuspage-exporter/pkg/config"
+	"github.com/sergeyshevch/statuspage-exporter/pkg/core"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/metrics"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/statusio"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/statuspageio"
@@ -75,8 +77,12 @@ func main() {
 	prometheus.MustRegister(metrics.ServiceStatus)
 	prometheus.MustRegister(metrics.ServiceStatusFetchError)
 
-	go statuspageio.StartFetchingLoop(ctx, wg, log)
-	go statusio.StartFetchingLoop(ctx, wg, log)
+	restyClient := resty.New().EnableTrace().SetTimeout(config.ClientTimeout()).SetRetryCount(config.RetryCount())
+
+	statusPageIOTargets, statusIOTargets := core.DetectStatusPageType(log, restyClient)
+
+	go core.StartFetchingLoop(ctx, wg, log, restyClient, statusPageIOTargets, statuspageio.FetchStatusPages)
+	go core.StartFetchingLoop(ctx, wg, log, restyClient, statusIOTargets, statusio.FetchStatusPages)
 	go startHTTP(ctx, wg, log)
 
 	quit := make(chan os.Signal, 1)

pkg/config/config.go

@@ -12,8 +12,9 @@ import (
 
 const (
 	defaultClientTimeout = 2 * time.Second
-	defaultFetchDelay    = 5 * time.Second
+	defaultFetchDelay    = 15 * time.Second
 	defaultHTTPPort      = 8080
+	defaultRetryCount    = 3
 )
 
 var configMutex = &sync.Mutex{}
@@ -73,19 +74,20 @@ func ClientTimeout() time.Duration {
 	return value
 }
 
-// StatusPageIoPages returns a list of status pages with statuspage.io engine to monitor.
-func StatusPageIoPages() []string {
+// RetryCount returns amount of retries for http client.
+func RetryCount() int {
 	configMutex.Lock()
-	value := viper.GetStringSlice("statuspageio_pages")
+	viper.SetDefault("retry_count", defaultRetryCount)
+	value := viper.GetInt("retry_count")
 	configMutex.Unlock()
 
 	return value
 }
 
-// StatusIoPages returns a list of status pages with status.io engine to monitor.
-func StatusIoPages() []string {
+// StatusPages returns a list of status pages to monitor.
+func StatusPages() []string {
 	configMutex.Lock()
-	value := viper.GetStringSlice("statusio_pages")
+	value := viper.GetStringSlice("statuspages")
 	configMutex.Unlock()
 
 	return value

pkg/core/detect.go

@@ -0,0 +1,29 @@
+package core
+
+import (
+	"github.com/go-resty/resty/v2"
+	"go.uber.org/zap"
+
+	"github.com/sergeyshevch/statuspage-exporter/pkg/config"
+	"github.com/sergeyshevch/statuspage-exporter/pkg/statusio"
+	"github.com/sergeyshevch/statuspage-exporter/pkg/statuspageio"
+)
+
+// DetectStatusPageType detects statuspage engine for all configured statuspage URLs.
+func DetectStatusPageType(log *zap.Logger, restyClient *resty.Client) ([]string, []string) {
+	targetUrls := config.StatusPages()
+
+	var statusPageIoPages, statusIoPages []string
+
+	for _, targetURL := range targetUrls {
+		if statuspageio.IsStatusPageIOPage(log, targetURL, restyClient) {
+			log.Info("Detected StatusPage.io page", zap.String("url", targetURL))
+			statusPageIoPages = append(statusPageIoPages, targetURL)
+		} else if statusio.IsStatusIOPage(log, targetURL, restyClient) {
+			log.Info("Detected Status.io page", zap.String("url", targetURL))
+			statusIoPages = append(statusIoPages, targetURL)
+		}
+	}
+
+	return statusPageIoPages, statusIoPages
+}

pkg/core/fetch.go

@@ -0,0 +1,36 @@
+package core
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/go-resty/resty/v2"
+	"go.uber.org/zap"
+
+	"github.com/sergeyshevch/statuspage-exporter/pkg/config"
+)
+
+// StartFetchingLoop starts a loop that fetches status pages.
+func StartFetchingLoop(ctx context.Context, wg *sync.WaitGroup,
+	log *zap.Logger, client *resty.Client, targetURLs []string,
+	fetcher func(*zap.Logger, *resty.Client, []string),
+) {
+	wg.Add(1)
+	defer wg.Done()
+
+	fetchDelay := config.FetchDelay()
+
+	for {
+		select {
+		default:
+			fetcher(log, client, targetURLs)
+
+			time.Sleep(fetchDelay)
+		case <-ctx.Done():
+			log.Info("Stopping fetching loop")
+
+			return
+		}
+	}
+}

pkg/statusio/fetch.go

@@ -1,63 +1,48 @@
 package statusio
 
 import (
-	"context"
 	"io"
 	"net/url"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/go-resty/resty/v2"
 	"go.uber.org/zap"
 
 	"github.com/PuerkitoBio/goquery"
 
-	"github.com/sergeyshevch/statuspage-exporter/pkg/config"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/metrics"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/utils"
 )
 
-// StartFetchingLoop starts a loop that fetches status pages.
-func StartFetchingLoop(ctx context.Context, wg *sync.WaitGroup, log *zap.Logger) {
-	wg.Add(1)
-	defer wg.Done()
-
-	fetchDelay := config.FetchDelay()
-	client := resty.New().EnableTrace().SetTimeout(config.ClientTimeout())
-
-	for {
-		select {
-		default:
-			fetchAllStatusPages(log, client)
-
-			time.Sleep(fetchDelay)
-		case <-ctx.Done():
-			log.Info("Stopping fetching loop")
+// IsStatusIOPage checks if given URL is Status.io page.
+func IsStatusIOPage(log *zap.Logger, targetURL string, client *resty.Client) bool {
+	parsedURL, err := constructURL(log, targetURL)
+	if err != nil {
+		return false
+	}
 
-			return
-		}
+	resp, err := client.R().Get(parsedURL)
+	if err != nil {
+		metrics.ServiceStatusFetchError.WithLabelValues(targetURL).Inc()
+		log.Error("Failed to check if page is Status.io page", zap.String("url", targetURL), zap.Error(err))
 	}
+
+	return strings.Contains(resp.String(), "status.io")
 }
 
-func fetchAllStatusPages(log *zap.Logger, client *resty.Client) {
+// FetchStatusPages fetch given status pages and export result as metrics.
+func FetchStatusPages(log *zap.Logger, client *resty.Client, targetURLs []string) {
 	wg := &sync.WaitGroup{}
 
-	targetUrls := config.StatusIoPages()
-
-	for _, targetURL := range targetUrls {
+	for _, targetURL := range targetURLs {
 		go fetchStatusPage(wg, log, targetURL, client)
 	}
 
 	wg.Wait()
 }
 
-func fetchStatusPage(wg *sync.WaitGroup, log *zap.Logger, targetURL string, client *resty.Client) {
-	wg.Add(1)
-	log.Info("Fetching status page", zap.String("url", targetURL))
-
-	defer wg.Done()
-
+func constructURL(log *zap.Logger, targetURL string) (string, error) {
 	parsedURL, err := url.Parse(targetURL)
 	if err != nil {
 		panic(err)
@@ -66,13 +51,28 @@ func fetchStatusPage(wg *sync.WaitGroup, log *zap.Logger, targetURL string, clie
 	parsedURL.Path = "/"
 
 	if parsedURL.Host == "" {
-		log.Error("Invalid URL. It won't be parsed. Check that your url contains scheme", zap.String("url", targetURL))
+		log.Error(utils.ErrInvalidURL.Error(), zap.String("url", targetURL))
+
+		return "", utils.ErrInvalidURL
+	}
+
+	return parsedURL.String(), nil
+}
+
+func fetchStatusPage(wg *sync.WaitGroup, log *zap.Logger, targetURL string, client *resty.Client) {
+	wg.Add(1)
+	log.Info("Fetching status page", zap.String("url", targetURL))
+
+	defer wg.Done()
+
+	parsedURL, err := constructURL(log, targetURL)
+	if err != nil {
 		metrics.ServiceStatusFetchError.WithLabelValues(targetURL).Inc()
 
 		return
 	}
 
-	resp, err := client.R().SetDoNotParseResponse(true).Get(parsedURL.String())
+	resp, err := client.R().SetDoNotParseResponse(true).Get(parsedURL)
 	if err != nil {
 		log.Error(
 			"Error fetching status page",

pkg/statuspageio/fetch.go

@@ -1,59 +1,44 @@
 package statuspageio
 
 import (
-	"context"
 	"net/url"
 	"sync"
-	"time"
 
 	"github.com/go-resty/resty/v2"
 	"go.uber.org/zap"
 
-	"github.com/sergeyshevch/statuspage-exporter/pkg/config"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/metrics"
 	"github.com/sergeyshevch/statuspage-exporter/pkg/utils"
 )
 
-// StartFetchingLoop starts a loop that fetches status pages.
-func StartFetchingLoop(ctx context.Context, wg *sync.WaitGroup, log *zap.Logger) {
-	wg.Add(1)
-	defer wg.Done()
-
-	fetchDelay := config.FetchDelay()
-	client := resty.New().EnableTrace().SetTimeout(config.ClientTimeout())
-
-	for {
-		select {
-		default:
-			fetchAllStatusPages(log, client)
-
-			time.Sleep(fetchDelay)
-		case <-ctx.Done():
-			log.Info("Stopping fetching loop")
+// IsStatusPageIOPage checks if given URL is StatusPage.io page.
+func IsStatusPageIOPage(log *zap.Logger, targetURL string, client *resty.Client) bool {
+	parsedURL, err := constructURL(log, targetURL)
+	if err != nil {
+		return false
+	}
 
-			return
-		}
+	resp, err := client.R().Head(parsedURL)
+	if err != nil {
+		metrics.ServiceStatusFetchError.WithLabelValues(targetURL).Inc()
+		log.Error("Failed to check if page is StatusPage.io page", zap.String("url", targetURL), zap.Error(err))
 	}
+
+	return resp.IsSuccess()
 }
 
-func fetchAllStatusPages(log *zap.Logger, client *resty.Client) {
+// FetchStatusPages fetch given status pages and export result as metrics.
+func FetchStatusPages(log *zap.Logger, client *resty.Client, targetUrls []string) {
 	wg := &sync.WaitGroup{}
 
-	targetUrls := config.StatusPageIoPages()
-
 	for _, targetURL := range targetUrls {
 		go fetchStatusPage(wg, log, targetURL, client)
 	}
 
 	wg.Wait()
 }
 
-func fetchStatusPage(wg *sync.WaitGroup, log *zap.Logger, targetURL string, client *resty.Client) {
-	wg.Add(1)
-	log.Info("Fetching status page", zap.String("url", targetURL))
-
-	defer wg.Done()
-
+func constructURL(log *zap.Logger, targetURL string) (string, error) {
 	parsedURL, err := url.Parse(targetURL)
 	if err != nil {
 		panic(err)
@@ -62,13 +47,28 @@ func fetchStatusPage(wg *sync.WaitGroup, log *zap.Logger, targetURL string, clie
 	parsedURL.Path = "/api/v2/components.json"
 
 	if parsedURL.Host == "" {
-		log.Error("Invalid URL. It won't be parsed. Check that your url contains scheme", zap.String("url", targetURL))
+		log.Error(utils.ErrInvalidURL.Error(), zap.String("url", targetURL))
+
+		return "", utils.ErrInvalidURL
+	}
+
+	return parsedURL.String(), nil
+}
+
+func fetchStatusPage(wg *sync.WaitGroup, log *zap.Logger, targetURL string, client *resty.Client) {
+	wg.Add(1)
+	log.Info("Fetching status page", zap.String("url", targetURL))
+
+	defer wg.Done()
+
+	parsedURL, err := constructURL(log, targetURL)
+	if err != nil {
 		metrics.ServiceStatusFetchError.WithLabelValues(targetURL).Inc()
 
 		return
 	}
 
-	resp, err := client.R().SetResult(&AtlassianStatusPageResponse{}).Get(parsedURL.String()) //nolint:exhaustruct
+	resp, err := client.R().SetResult(&AtlassianStatusPageResponse{}).Get(parsedURL) //nolint:exhaustruct
 	if err != nil {
 		log.Error(
 			"Error fetching status page",

pkg/utils/errors.go

@@ -0,0 +1,6 @@
+package utils
+
+import "errors"
+
+// ErrInvalidURL is returned when URL is invalid.
+var ErrInvalidURL = errors.New("invalid URL. It won't be parsed. Check that your url contains scheme")