feat: capture the code coverage of the go binaries in test (#1171)

* feat: captures code coverage for the go binaries

Signed-off-by: re-Tick <jain.ritik.1001@gmail.com>

---------

Signed-off-by: re-Tick <jain.ritik.1001@gmail.com>

cmd/test.go

@@ -38,7 +38,7 @@ func readTestConfig(configPath string) (*models.Test, error) {
 	return &doc.Test, nil
 }
 
-func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, testsets *[]string, appContainer, networkName *string, Delay *uint64, passThorughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, configPath string) error {
+func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, testsets *[]string, appContainer, networkName *string, Delay *uint64, passThorughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string) error {
 	configFilePath := filepath.Join(configPath, "keploy-config.yaml")
 	if isExist := utils.CheckFileExists(configFilePath); !isExist {
 		return errFileNotFound
@@ -70,7 +70,11 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te
 	}
 	if len(*passThorughPorts) == 0 {
 		*passThorughPorts = confTest.PassThroughPorts
+	} 
+	if len(*coverageReportPath) == 0 {
+		*coverageReportPath = confTest.CoverageReportPath
 	}
+	*withCoverage = *withCoverage || confTest.WithCoverage
 	if *apiTimeout == 5 {
 		*apiTimeout = confTest.ApiTimeout
 	}
@@ -143,6 +147,16 @@ func (t *Test) GetCmd() *cobra.Command {
 				t.logger.Error("failed to read the testcase path input")
 				return err
 			}
+			withCoverage, err := cmd.Flags().GetBool("withCoverage")
+			if err != nil {
+				t.logger.Error("failed to read the go coverage binary", zap.Error(err))
+				return err
+			}
+			coverageReportPath, err := cmd.Flags().GetString("coverageReportPath")
+			if err != nil {
+				t.logger.Error("failed to read the go coverage directory path", zap.Error(err))
+				return err
+			}
 
 			appCmd, err := cmd.Flags().GetString("command")
 			if err != nil {
@@ -201,7 +215,7 @@ func (t *Test) GetCmd() *cobra.Command {
 			globalNoise := make(models.GlobalNoise)
 			testsetNoise := make(models.TestsetNoise)
 
-			err = t.getTestConfig(&path, &proxyPort, &appCmd, &testSets, &appContainer, &networkName, &delay, &ports, &apiTimeout, &globalNoise, &testsetNoise, configPath)
+			err = t.getTestConfig(&path, &proxyPort, &appCmd, &testSets, &appContainer, &networkName, &delay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath)
 			if err != nil {
 				if err == errFileNotFound {
 					t.logger.Info("continuing without configuration file because file not found")
@@ -274,16 +288,18 @@ func (t *Test) GetCmd() *cobra.Command {
 			t.logger.Debug("the configuration for mocking mongo connection", zap.Any("password", mongoPassword))
 
 			t.tester.Test(path, testReportPath, appCmd, test.TestOptions{
-				Testsets:         testSets,
-				AppContainer:     appContainer,
-				AppNetwork:       networkName,
-				MongoPassword:    mongoPassword,
-				Delay:            delay,
-				PassThroughPorts: ports,
-				ApiTimeout:       apiTimeout,
-				ProxyPort:        proxyPort,
-				GlobalNoise:      globalNoise,
-				TestsetNoise:     testsetNoise,
+				Testsets:           testSets,
+				AppContainer:       appContainer,
+				AppNetwork:         networkName,
+				MongoPassword:      mongoPassword,
+				Delay:              delay,
+				PassThroughPorts:   ports,
+				ApiTimeout:         apiTimeout,
+				ProxyPort:          proxyPort,
+				GlobalNoise:        globalNoise,
+				TestsetNoise:       testsetNoise,
+				WithCoverage:       withCoverage,
+				CoverageReportPath: coverageReportPath,
 			})
 
 			return nil
@@ -311,6 +327,10 @@ func (t *Test) GetCmd() *cobra.Command {
 
 	testCmd.Flags().String("mongoPassword", "default123", "Authentication password for mocking MongoDB connection")
 
+	testCmd.Flags().String("coverageReportPath", "", "Write a go coverage profile to the file in the given directory.")
+
+	testCmd.Flags().Bool("withCoverage", false, "Capture the code coverage of the go binary in the command flag.")
+	testCmd.Flags().Lookup("withCoverage").NoOptDefVal = "true"
 	testCmd.SilenceUsage = true
 	testCmd.SilenceErrors = true
 

pkg/hooks/loader.go

@@ -321,7 +321,7 @@ func (h *Hook) killProcessesAndTheirChildren(parentPID int) {
 
 	for _, childPID := range pids {
 		if h.userAppCmd.ProcessState == nil {
-			err := syscall.Kill(childPID, syscall.SIGKILL)
+			err := syscall.Kill(childPID, syscall.SIGTERM)
 			if err != nil {
 				h.logger.Error("failed to set kill child pid", zap.Any("error killing child process", err.Error()))
 			}

pkg/models/config.go

@@ -20,16 +20,18 @@ type Filters struct {
 	URLMethods map[string][]string `json:"urlMethods" yaml:"urlMethods"`
 }
 type Test struct {
-	Path             string   `json:"path" yaml:"path"`
-	Command          string   `json:"command" yaml:"command"`
-	ProxyPort        uint32   `json:"proxyport" yaml:"proxyport"`
-	ContainerName    string   `json:"containerName" yaml:"containerName"`
-	NetworkName      string   `json:"networkName" yaml:"networkName"`
-	TestSets         []string `json:"testSets" yaml:"testSets"`
-	GlobalNoise      string   `json:"globalNoise" yaml:"globalNoise"`
-	Delay            uint64   `json:"delay" yaml:"delay"`
-	ApiTimeout       uint64   `json:"apiTimeout" yaml:"apiTimeout"`
-	PassThroughPorts []uint   `json:"passThroughPorts" yaml:"passThroughPorts"`
+	Path               string   `json:"path" yaml:"path"`
+	Command            string   `json:"command" yaml:"command"`
+	ProxyPort          uint32   `json:"proxyport" yaml:"proxyport"`
+	ContainerName      string   `json:"containerName" yaml:"containerName"`
+	NetworkName        string   `json:"networkName" yaml:"networkName"`
+	TestSets           []string `json:"testSets" yaml:"testSets"`
+	GlobalNoise        string   `json:"globalNoise" yaml:"globalNoise"`
+	Delay              uint64   `json:"delay" yaml:"delay"`
+	ApiTimeout         uint64   `json:"apiTimeout" yaml:"apiTimeout"`
+	PassThroughPorts   []uint   `json:"passThroughPorts" yaml:"passThroughPorts"`
+	WithCoverage       bool     `json:"withCoverage" yaml:"withCoverage"` // boolean to capture the coverage in test
+	CoverageReportPath string   `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files
 }
 
 type (

pkg/service/generateConfig/generateConfig.go

@@ -60,6 +60,8 @@ test:
   delay: 5
   apiTimeout: 5
   passThroughPorts: []
+  withCoverage: false
+  coverageReportPath: ""
   #
   # Example on using globalNoise
   # globalNoise: |-

pkg/service/test/test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"strings"
@@ -35,16 +36,18 @@ type tester struct {
 	mutex  sync.Mutex
 }
 type TestOptions struct {
-	MongoPassword    string
-	Delay            uint64
-	PassThroughPorts []uint
-	ApiTimeout       uint64
-	Testsets         []string
-	AppContainer     string
-	AppNetwork       string
-	ProxyPort        uint32
-	GlobalNoise      models.GlobalNoise
-	TestsetNoise     models.TestsetNoise
+	MongoPassword      string
+	Delay              uint64
+	PassThroughPorts   []uint
+	ApiTimeout         uint64
+	Testsets           []string
+	AppContainer       string
+	AppNetwork         string
+	ProxyPort          uint32
+	GlobalNoise        models.GlobalNoise
+	TestsetNoise       models.TestsetNoise
+	WithCoverage       bool
+	CoverageReportPath string
 }
 
 func NewTester(logger *zap.Logger) Tester {
@@ -57,6 +60,50 @@ func NewTester(logger *zap.Logger) Tester {
 func (t *tester) InitialiseTest(cfg *TestConfig) (InitialiseTestReturn, error) {
 	var returnVal InitialiseTestReturn
 
+	// capturing the code coverage for go bianries built by go-version 1.20^
+	if cfg.WithCoverage {
+
+		// report path is provided via cmd flag by user
+		if cfg.CoverageReportPath != "" {
+
+			// handle relative path
+			if !strings.HasPrefix(cfg.CoverageReportPath, "/") {
+				absPath, err := filepath.Abs(cfg.CoverageReportPath)
+				if err != nil {
+					t.logger.Error("failed to resolve the relative path for go coverage directory", zap.Error(err), zap.Any("relative path", cfg.CoverageReportPath))
+				}
+				cfg.CoverageReportPath = absPath
+			}
+			cfg.CoverageReportPath = cfg.CoverageReportPath + "/coverage-reports"
+
+			// validate the path is to directory or not. And create a directory if not exists
+			dirInfo, err := os.Stat(cfg.CoverageReportPath)
+			if err != nil && !os.IsNotExist(err) {
+				t.logger.Error("failed to check that the goCoverDir path is a directory", zap.Error(err))
+				return returnVal, err
+			} else if err == nil && !dirInfo.IsDir() {
+				t.logger.Error("the goCoverDir is not a directory. Please provide a valid path to a directory for go coverage binaries.")
+				return returnVal, fmt.Errorf("the goCoverDir is not a directory. Please provide a valid path to a directory for go coverage binaries.")
+			} else if err != nil && os.IsNotExist(err) {
+				err := makeDirectory(cfg.CoverageReportPath)
+				if err != nil {
+					t.logger.Error("failed to create coverage directory to collect the go coverage", zap.Error(err), zap.Any("path", cfg.CoverageReportPath))
+					return returnVal, err
+				}
+			}
+		} else {
+			// reports at the current directory
+			cfg.CoverageReportPath = cfg.Path + "/coverage-reports"
+			err := makeDirectory(cfg.CoverageReportPath)
+			if err != nil {
+				t.logger.Error("failed to create coverage directory to collect the go coverage", zap.Error(err), zap.Any("path", cfg.CoverageReportPath))
+				return returnVal, err
+			}
+		}
+		// set the go env variable to export the coverage-path of the runnable binaries
+		os.Setenv("GOCOVERDIR", cfg.CoverageReportPath)
+	}
+
 	stopper := make(chan os.Signal, 1)
 	signal.Notify(stopper, os.Interrupt, os.Kill, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
 
@@ -157,17 +204,19 @@ func (t *tester) Test(path string, testReportPath string, appCmd string, options
 	exitLoop := false
 
 	cfg := &TestConfig{
-		Path:             path,
-		Proxyport:        options.ProxyPort,
-		TestReportPath:   testReportPath,
-		AppCmd:           appCmd,
-		Testsets:         &options.Testsets,
-		AppContainer:     options.AppContainer,
-		AppNetwork:       options.AppContainer,
-		Delay:            options.Delay,
-		PassThroughPorts: options.PassThroughPorts,
-		ApiTimeout:       options.ApiTimeout,
-		MongoPassword:    options.MongoPassword,
+		Path:               path,
+		Proxyport:          options.ProxyPort,
+		TestReportPath:     testReportPath,
+		AppCmd:             appCmd,
+		Testsets:           &options.Testsets,
+		AppContainer:       options.AppContainer,
+		AppNetwork:         options.AppContainer,
+		Delay:              options.Delay,
+		PassThroughPorts:   options.PassThroughPorts,
+		ApiTimeout:         options.ApiTimeout,
+		MongoPassword:      options.MongoPassword,
+		WithCoverage:       options.WithCoverage,
+		CoverageReportPath: options.CoverageReportPath,
 	}
 	initialisedValues, err := t.InitialiseTest(cfg)
 	// Recover from panic and gracfully shutdown
@@ -208,7 +257,28 @@ func (t *tester) Test(path string, testReportPath string, appCmd string, options
 		}
 	}
 	t.logger.Info("test run completed", zap.Bool("passed overall", result))
-
+	// log the overall code coverage for the test run of go binaries
+	if options.WithCoverage {
+		t.logger.Info("there is a opportunity to get the coverage here")
+		// logs the coverage using covdata 
+		coverCmd := exec.Command("go", "tool", "covdata", "percent", "-i="+os.Getenv("GOCOVERDIR"))
+		output, err := coverCmd.Output()
+		if err != nil {
+			t.logger.Error("failed to get the coverage of the go binary", zap.Error(err), zap.Any("cmd", coverCmd.String()))
+		}
+		t.logger.Sugar().Infoln("\n", models.HighlightPassingString(string(output)))
+
+		// merges the coverage files into a single txt file which can be merged with the go-test coverage
+		generateCovTxtCmd := exec.Command("go", "tool", "covdata", "textfmt", "-i="+os.Getenv("GOCOVERDIR"), "-o="+os.Getenv("GOCOVERDIR")+"/total-coverage.txt")
+		output, err = generateCovTxtCmd.Output()
+		if err != nil {
+			t.logger.Error("failed to get the coverage of the go binary", zap.Error(err), zap.Any("cmd", coverCmd.String()))
+		}
+		if len(output) > 0 {
+			t.logger.Sugar().Infoln("\n", models.HighlightFailingString(string(output)))
+		}
+	}
+
 	if !initialisedValues.AbortStopHooksForcefully {
 		initialisedValues.AbortStopHooksInterrupt <- true
 		// stop listening for the eBPF events

pkg/service/test/util.go

@@ -5,10 +5,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"go.keploy.io/server/pkg/hooks"
@@ -53,6 +55,8 @@ type TestConfig struct {
 	Delay            uint64
 	PassThroughPorts []uint
 	ApiTimeout       uint64
+	WithCoverage  bool
+	CoverageReportPath       string
 }
 
 type RunTestSetConfig struct {
@@ -401,3 +405,14 @@ func FilterTcsMocks(tc *models.TestCase, m []*models.Mock, logger *zap.Logger) [
 	}
 	return filteredMocks
 }
+
+// creates a directory if not exists with all user access
+func makeDirectory (path string) error {
+	oldUmask := syscall.Umask(0)
+	err := os.MkdirAll(path, 0777)
+	if err != nil {
+		return err
+	}
+	syscall.Umask(oldUmask)
+	return nil
+}