fix(tests): refactor testscript tests

cmd/karma/main.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"io/ioutil"
 	"mime"
 	"net"
 	"net/http"
 	"os"
 	"os/signal"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -54,6 +56,8 @@ var (
 	protectedEndpoints *gin.RouterGroup
 
 	silenceACLs = []*silenceACL{}
+
+	pidFile string
 )
 
 func getViewURL(sub string) string {
@@ -252,6 +256,7 @@ func mainSetup(errorHandling pflag.ErrorHandling) (*gin.Engine, error) {
 	f := pflag.NewFlagSet("karma", errorHandling)
 	printVersion := f.Bool("version", false, "Print version and exit")
 	validateConfig := f.Bool("check-config", false, "Validate configuration and exit")
+	f.StringVar(&pidFile, "pid-file", "", "If set PID of karma process will be written to this file")
 	config.SetupFlags(f)
 
 	err := f.Parse(os.Args[1:])
@@ -264,7 +269,11 @@ func mainSetup(errorHandling pflag.ErrorHandling) (*gin.Engine, error) {
 		return nil, nil
 	}
 
-	configFile := config.Config.Read(f)
+	configFile, err := config.Config.Read(f)
+	if err != nil {
+		return nil, err
+	}
+
 	err = setupLogger()
 	if err != nil {
 		return nil, err
@@ -368,13 +377,41 @@ func mainSetup(errorHandling pflag.ErrorHandling) (*gin.Engine, error) {
 	return router, nil
 }
 
-func main() {
-	router, err := mainSetup(pflag.ExitOnError)
+func writePidFile() error {
+	if pidFile != "" {
+		log.Infof("Writing PID file to %q", pidFile)
+		pid := os.Getpid()
+		err := ioutil.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0644)
+		if err != nil {
+			return fmt.Errorf("Failed to write a PID file: %s", err)
+		}
+	}
+	return nil
+}
+
+func removePidFile() error {
+	if pidFile != "" {
+		log.Infof("Removing PID file %q", pidFile)
+		err := os.Remove(pidFile)
+		if err != nil {
+			return fmt.Errorf("Failed to remove PID file: %s", err)
+		}
+	}
+	return nil
+}
+
+func serve(errorHandling pflag.ErrorHandling) error {
+	router, err := mainSetup(errorHandling)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	if router == nil {
-		return
+		return nil
+	}
+
+	err = writePidFile()
+	if err != nil {
+		return err
 	}
 
 	// before we start try to fetch data from Alertmanager
@@ -389,7 +426,7 @@ func main() {
 	listen := fmt.Sprintf("%s:%d", config.Config.Listen.Address, config.Config.Listen.Port)
 	listener, err := net.Listen("tcp", listen)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	log.Infof("Listening on %s", listener.Addr())
 
@@ -409,7 +446,16 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	if err := httpServer.Shutdown(ctx); err != nil {
-		log.Fatalf("Shutdown failed: %s", err)
+		return fmt.Errorf("Shutdown failed: %s", err)
 	}
 	log.Info("HTTP server shut down")
+
+	return removePidFile()
+}
+
+func main() {
+	err := serve(pflag.ExitOnError)
+	if err != nil {
+		log.Fatal(err)
+	}
 }

cmd/karma/script_test.go

@@ -11,25 +11,13 @@ import (
 )
 
 func mainShoulFail() int {
-	var wasFatal bool
-	defer func() {
-		if r := recover(); r != nil {
-			wasFatal = true
-		}
-	}()
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
-
-	_, err := mainSetup(pflag.ContinueOnError)
+	err := serve(pflag.ContinueOnError)
 	if err != nil {
 		log.Error(err)
-	} else if wasFatal {
 		return 0
-	} else {
-		log.Error("No error logged")
-		return 100
 	}
-	return 0
+	log.Error("No error logged")
+	return 100
 }
 
 func mainShoulFailNoTimestamp() int {
@@ -40,7 +28,7 @@ func mainShoulFailNoTimestamp() int {
 }
 
 func mainShouldWork() int {
-	_, err := mainSetup(pflag.ContinueOnError)
+	err := serve(pflag.ContinueOnError)
 	if err != nil {
 		log.Error(err)
 		return 100

cmd/karma/tests/testscript/035_debug.txt

@@ -0,0 +1,12 @@
+# Passing --debug enables Gin debug mode
+
+exec sh -ex ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --log.format=text --log.config=false --debug --alertmanager.uri=http://localhost --listen.address=127.0.0.1 --listen.port=8035
+stdout '\[GIN-debug\] \[WARNING\] Running in "debug" mode. Switch to "release" mode in production.'
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/invalid_config_alertmanager_timeout.txt → cmd/karma/tests/testscript/042_invalid_config_alertmanager_timeout.txt

@@ -1,7 +1,7 @@
 # Raises an error if we pass alertmanager.interval value that doesn't parse
 karma.bin-should-fail-no-timestamp --log.format=text --log.config=false --log.level=error --config.file karma.yaml
 ! stdout .
-stderr 'level=fatal msg=".* invalid duration \\"abc123\\""'
+stderr 'msg=".* invalid duration \\"abc123\\""'
 
 -- karma.yaml --
 alertmanager:

cmd/karma/tests/testscript/061_log_full_config_file_invalid_values.txt

@@ -0,0 +1,59 @@
+# Print out and compare logged config set via config file that includes invalid values
+karma.bin-should-fail-no-timestamp --config.file=karma.yaml --check-config
+! stdout .
+stderr 'Failed to unmarshal configuration: 12 error\(s\) decoding:'
+
+-- karma.yaml --
+alertmanager:
+  interval: jjs88
+  servers:
+    - name: ha1
+      uri: "http://localhost:9093"
+      timeout: bbb
+      proxy: YEs
+      cors:
+        credentials: foo
+    - name: ha2
+      uri: "http://localhost:9094"
+      timeout: 11
+      readonly: 1
+    - name: local
+      uri: http://localhost:9095
+      timeout: z
+      proxy: true
+      readonly: 0
+      headers:
+        - X-Auth-Test=some-token-or-other-string
+    - name: client-auth
+      uri: https://localhost:9096
+      timeout: 10s
+      tls:
+        ca: ca.pem
+        cert: cert.pem
+        key: key.pem
+alertAcknowledgement:
+  enabled: zzz
+  duration: 7m0s
+  author: karma
+  commentPrefix: ACK!
+annotations:
+  default:
+    hidden: z
+  hidden: {}
+  visible:
+    - visible
+filters:
+  default: []
+karma:
+  name: karma-demo
+log:
+  level: 123
+  format: foo
+ui:
+  refresh: 10sm
+  hideFiltersWhenIdle: z
+  colorTitlebar: yum
+  theme: x
+  minimalGroupWidth: abc4
+  alertsPerGroup: 5a
+  collapseGroups: collapsedOanMobile

cmd/karma/tests/testscript/063_missing_config_file.txt

@@ -0,0 +1,4 @@
+# Errors when config.file points to missing file
+karma.bin-should-fail --config.file=404.yaml
+! stdout .
+stderr 'msg="Failed to load configuration file \\"404.yaml\\": open 404.yaml: no such file or directory'

cmd/karma/tests/testscript/068_sentry.txt

@@ -0,0 +1,23 @@
+# Configures sentry when enabled
+
+exec sh -ex ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --log.format=text --log.config=true --config.file=karma.yaml --listen.address=127.0.0.1 --listen.port=8068
+! stdout .
+stderr 'msg="  private: secret"'
+stderr 'msg="  public: \\"123456789\\""'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: http://localhost:9093
+sentry:
+  private: secret
+  public: 123456789
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/069_simple_config.txt

@@ -0,0 +1,13 @@
+# Works in simple mode when single --alertmanager.uri flag is passed
+
+exec sh -ex ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --log.format=text --log.config=false --alertmanager.uri=http://localhost --listen.address=127.0.0.1 --listen.port=8069
+! stdout .
+stderr 'msg="\[default\] Configured Alertmanager source at http://localhost \(proxied: false\, readonly: false\)"'
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/072_pid_file_write_error.txt

@@ -0,0 +1,5 @@
+# Fails is we cannot write a PID file
+
+karma.bin-should-fail --pid-file=/foo/bar/karma.pid --log.format=text --log.config=false --alertmanager.uri=http://localhost
+! stdout .
+stderr 'msg="Failed to write a PID file:'

cmd/karma/tests/testscript/073_pid_file_remove_error.txt

@@ -0,0 +1,15 @@
+# Fails is we cannot remove a PID file
+
+exec sh -ex ./test.sh &
+karma.bin-should-fail --pid-file=karma.pid --log.format=text --log.config=false --alertmanager.uri=http://localhost --listen.address=127.0.0.1 --listen.port=8073
+! stdout .
+stderr 'msg="Failed to remove PID file:'
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+PID=$(cat karma.pid)
+rm karma.pid
+kill $PID