Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature: added generateTestReport flag #1513

Merged
merged 13 commits into from
Feb 11, 2024
Merged

Feature: added generateTestReport flag #1513

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c112f06
added generateTestReport flag
Yaxhveer Feb 6, 2024
bede2c3
error fixed
Yaxhveer Feb 6, 2024
26befdd
error fixed
Yaxhveer Feb 7, 2024
4a2ce9d
delete testReports in serve command
Yaxhveer Feb 7, 2024
b41fb13
error fixed
Yaxhveer Feb 7, 2024
9a04746
fixed issues
Yaxhveer Feb 7, 2024
6c669db
Merge branch 'main' into test-report-optional
Yaxhveer Feb 7, 2024
fa4a8e3
changes updated
Yaxhveer Feb 7, 2024
be20fed
Merge branch 'main' into test-report-optional
Yaxhveer Feb 7, 2024
26f55a9
changes updated
Yaxhveer Feb 7, 2024
fde1d52
Merge branch 'main' into test-report-optional
Yaxhveer Feb 8, 2024
2177e52
removed extra logs
Yaxhveer Feb 9, 2024
c9bd4e5
Merge branch 'keploy:main' into test-report-optional
Yaxhveer Feb 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions cmd/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func ReadTestConfig(configPath string) (*models.Test, error) {
return &doc.Test, nil
}

func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool, passThroughHosts *[]models.Filters) error {
func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, generateTestReport *bool, configPath string, ignoreOrdering *bool, passThroughHosts *[]models.Filters) error {
configFilePath := filepath.Join(configPath, "keploy-config.yaml")
if isExist := utils.CheckFileExists(configFilePath); !isExist {
return errFileNotFound
Expand Down Expand Up @@ -82,6 +82,7 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te
*coverageReportPath = confTest.CoverageReportPath
}
*withCoverage = *withCoverage || confTest.WithCoverage
*generateTestReport = *generateTestReport || confTest.GenerateTestReport
if *apiTimeout == 5 {
*apiTimeout = confTest.ApiTimeout
}
Expand Down Expand Up @@ -221,6 +222,12 @@ func (t *Test) GetCmd() *cobra.Command {
return err
}

generateTestReport, err := cmd.Flags().GetBool("generateTestReport")
if err != nil {
t.logger.Error("failed to read the generate test teport flag")
return err
}

enableTele, err := cmd.Flags().GetBool("enableTele")
if err != nil {
t.logger.Error("failed to read the disable telemetry flag")
Expand Down Expand Up @@ -250,7 +257,7 @@ func (t *Test) GetCmd() *cobra.Command {

passThroughHosts := []models.Filters{}

err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering, &passThroughHosts)
err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, &generateTestReport, configPath, &ignoreOrdering, &passThroughHosts)
if err != nil {
if err == errFileNotFound {
t.logger.Info("Keploy config not found, continuing without configuration")
Expand Down Expand Up @@ -384,17 +391,21 @@ func (t *Test) GetCmd() *cobra.Command {
}

path += "/keploy"
t.logger.Info("", zap.Any("keploy test and mock path", path))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why has this line been added

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually t.logger.Info("", zap.Any("keploy test and mock path", path), zap.Any("keploy testReport path", testReportPath)) was a combined log and I have divided it into two different log for path and testReport Path.


testReportPath := path + "/testReports"
testReportPath := ""

testReportPath, err = pkg.GetNextTestReportDir(testReportPath, models.TestRunTemplateName)
if err != nil {
t.logger.Error("failed to get the next test report directory", zap.Error(err))
return err
if generateTestReport {
testReportPath = path + "/testReports"

testReportPath, err = pkg.GetNextTestReportDir(testReportPath, models.TestRunTemplateName)
t.logger.Info("", zap.Any("keploy testReport path", testReportPath))
if err != nil {
t.logger.Error("failed to get the next test report directory", zap.Error(err))
return err
}
}

t.logger.Info("", zap.Any("keploy test and mock path", path), zap.Any("keploy testReport path", testReportPath))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why have these essential logs been removed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar reason divided it into two different logs



var hasContainerName bool
if isDockerCmd {
if strings.Contains(appCmd, "--name") {
Expand All @@ -415,10 +426,10 @@ func (t *Test) GetCmd() *cobra.Command {
t.logger.Debug("the configuration for mocking mongo connection", zap.Any("password", mongoPassword))
if coverage {
g := graph.NewGraph(t.logger)
g.Serve(path, proxyPort, mongoPassword, testReportPath, delay, pid, port, lang, ports, apiTimeout, appCmd, enableTele)
g.Serve(path, proxyPort, mongoPassword, testReportPath, generateTestReport, delay, pid, port, lang, ports, apiTimeout, appCmd, enableTele)
} else {

t.tester.StartTest(path, testReportPath, appCmd, test.TestOptions{
t.tester.StartTest(path, testReportPath, generateTestReport, appCmd, test.TestOptions{
Tests: tests,
AppContainer: appContainer,
AppNetwork: networkName,
Expand Down Expand Up @@ -472,6 +483,8 @@ func (t *Test) GetCmd() *cobra.Command {

testCmd.Flags().Uint32("pid", 0, "Process id of your application.")

testCmd.Flags().BoolP("generateTestReport", "g", true, "Generate the test report")

testCmd.Flags().Bool("enableTele", true, "Switch for telemetry")

testCmd.Flags().Bool("ignoreOrdering", true, "Ignore ordering of array in response")
Expand All @@ -488,4 +501,4 @@ func (t *Test) GetCmd() *cobra.Command {
testCmd.SilenceErrors = true

return testCmd
}
}
23 changes: 12 additions & 11 deletions pkg/graph/resolver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,16 @@ import (
var Emoji = "\U0001F430" + " Keploy:"

type Resolver struct {
Tester test.Tester
TestReportFS platform.TestReportDB
Storage platform.TestCaseDB
LoadedHooks *hooks.Hook
Logger *zap.Logger
Path string
TestReportPath string
Delay uint64
AppPid uint32
ApiTimeout uint64
ServeTest bool
Tester test.Tester
TestReportFS platform.TestReportDB
Storage platform.TestCaseDB
LoadedHooks *hooks.Hook
Logger *zap.Logger
Path string
TestReportPath string
GenerateTestReport bool
Delay uint64
AppPid uint32
ApiTimeout uint64
ServeTest bool
}
2 changes: 1 addition & 1 deletion pkg/graph/schema.resolvers.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 15 additions & 12 deletions pkg/graph/serve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@ func NewGraph(logger *zap.Logger) graphInterface {
const defaultPort = 6789

// Serve is called by the serve command and is used to run a graphql server, to run tests separately via apis.
func (g *graph) Serve(path string, proxyPort uint32, mongopassword, testReportPath string, Delay uint64, pid, port uint32, lang string, passThroughPorts []uint, apiTimeout uint64, appCmd string, enableTele bool) {
func (g *graph) Serve(path string, proxyPort uint32, mongopassword, testReportPath string, generateTestReport bool, Delay uint64, pid, port uint32, lang string, passThroughPorts []uint, apiTimeout uint64, appCmd string, enableTele bool) {
var ps *proxy.ProxySet

defer pkg.DeleteTestReports(g.logger, generateTestReport)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why exactly are we deleting the reports at the end of the function?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

serve command require testReports to show status. So I removed the testReports when function ends

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we just get the status without writing to a file, or checking the condition before writing to the file?

Copy link
Contributor Author

@Yaxhveer Yaxhveer Feb 8, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually this approach was recommended here #1513 (comment).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok


if port == 0 {
port = defaultPort
}
Expand Down Expand Up @@ -112,17 +114,18 @@ func (g *graph) Serve(path string, proxyPort uint32, mongopassword, testReportPa

srv := handler.NewDefaultServer(NewExecutableSchema(Config{
Resolvers: &Resolver{
Tester: tester,
TestReportFS: testReportFS,
Storage: ys,
LoadedHooks: loadedHooks,
Logger: g.logger,
Path: path,
TestReportPath: testReportPath,
Delay: Delay,
AppPid: pid,
ApiTimeout: apiTimeout,
ServeTest: len(appCmd) != 0,
Tester: tester,
TestReportFS: testReportFS,
Storage: ys,
LoadedHooks: loadedHooks,
Logger: g.logger,
Path: path,
TestReportPath: testReportPath,
GenerateTestReport: generateTestReport,
Delay: Delay,
AppPid: pid,
ApiTimeout: apiTimeout,
ServeTest: len(appCmd) != 0,
},
}))

Expand Down
2 changes: 1 addition & 1 deletion pkg/graph/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ import (
)

type graphInterface interface {
Serve(path string, proxyPort uint32, mongoPassword, testReportPath string, Delay uint64, pid, port uint32, lang string, passThroughPorts []uint, apiTimeout uint64, appCmd string, enableTele bool)
Serve(path string, proxyPort uint32, mongoPassword, testReportPath string, generateTestReport bool, Delay uint64, pid, port uint32, lang string, passThroughPorts []uint, apiTimeout uint64, appCmd string, enableTele bool)
stopGraphqlServer(http *http.Server)
}
2 changes: 2 additions & 0 deletions pkg/models/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ test:
host: ""
ports: 0
withCoverage: false
generateTestReport: true
coverageReportPath: ""
`

Expand Down Expand Up @@ -102,6 +103,7 @@ type Test struct {
BypassEndpointsRegistry []string `json:"bypassEndpointsRegistry" yaml:"bypassEndpointsRegistry"`
WithCoverage bool `json:"withCoverage" yaml:"withCoverage"` // boolean to capture the coverage in test
CoverageReportPath string `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files
GenerateTestReport bool `json:"generateTestReport" yaml:"generateTestReport"`
IgnoreOrdering bool `json:"ignoreOrdering" yaml:"ignoreOrdering"`
Stubs Stubs `json:"stubs" yaml:"stubs"`
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/service/test/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ import (
)

type Tester interface {
Test(path string, testReportPath string, appCmd string, options TestOptions, tele *telemetry.Telemetry, testReportStorage platform.TestReportDB, tcsStorage platform.TestCaseDB) bool
StartTest(path string, testReportPath string, appCmd string, options TestOptions, enableTele bool) bool
RunTestSet(testSet, path, testReportPath, appCmd, appContainer, appNetwork string, delay uint64, buildDelay time.Duration, pid uint32, testRunChan chan string, apiTimeout uint64, testcases map[string]bool, noiseConfig models.GlobalNoise, serveTest bool, testEnv TestEnvironmentSetup) models.TestRunStatus
Test(path string, testReportPath string, generateTestReport bool, appCmd string, options TestOptions, tele *telemetry.Telemetry, testReportStorage platform.TestReportDB, tcsStorage platform.TestCaseDB) bool
StartTest(path string, testReportPath string, generateTestReport bool, appCmd string, options TestOptions, enableTele bool) bool
RunTestSet(testSet, path, testReportPath string, generateTestReport bool, appCmd, appContainer, appNetwork string, delay uint64, buildDelay time.Duration, pid uint32, testRunChan chan string, apiTimeout uint64, testcases map[string]bool, noiseConfig models.GlobalNoise, serveTest bool, testEnv TestEnvironmentSetup) models.TestRunStatus
InitialiseTest(cfg *TestConfig) (TestEnvironmentSetup, error)
InitialiseRunTestSet(cfg *RunTestSetConfig) InitialiseRunTestSetReturn
SimulateRequest(cfg *SimulateRequestConfig)
Expand Down
89 changes: 49 additions & 40 deletions pkg/service/test/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func (t *tester) InitialiseTest(cfg *TestConfig) (TestEnvironmentSetup, error) {
return returnVal, nil
}

func (t *tester) Test(path string, testReportPath string, appCmd string, options TestOptions, tele *telemetry.Telemetry, testReportStorage platform.TestReportDB, tcsStorage platform.TestCaseDB) bool {
func (t *tester) Test(path string, testReportPath string, generateTestReport bool, appCmd string, options TestOptions, tele *telemetry.Telemetry, testReportStorage platform.TestReportDB, tcsStorage platform.TestCaseDB) bool {

testRes := false
result := true
Expand All @@ -197,6 +197,7 @@ func (t *tester) Test(path string, testReportPath string, appCmd string, options
Path: path,
Proxyport: options.ProxyPort,
TestReportPath: testReportPath,
GenerateTestReport: generateTestReport,
AppCmd: appCmd,
AppContainer: options.AppContainer,
AppNetwork: options.AppContainer,
Expand Down Expand Up @@ -236,7 +237,7 @@ func (t *tester) Test(path string, testReportPath string, appCmd string, options
noiseConfig = LeftJoinNoise(options.GlobalNoise, tsNoise)
}

testRunStatus := t.RunTestSet(sessionIndex, path, testReportPath, appCmd, options.AppContainer, options.AppNetwork, options.Delay, options.BuildDelay, 0, nil, options.ApiTimeout, testcases, noiseConfig, false, initialisedValues)
testRunStatus := t.RunTestSet(sessionIndex, path, testReportPath, generateTestReport, appCmd, options.AppContainer, options.AppNetwork, options.Delay, options.BuildDelay, 0, nil, options.ApiTimeout, testcases, noiseConfig, false, initialisedValues)

switch testRunStatus {
case models.TestRunStatusAppHalted:
Expand Down Expand Up @@ -293,12 +294,12 @@ func (t *tester) Test(path string, testReportPath string, appCmd string, options
return false
}

func (t *tester) StartTest(path string, testReportPath string, appCmd string, options TestOptions, enableTele bool) bool {
func (t *tester) StartTest(path string, testReportPath string, generateTestReport bool, appCmd string, options TestOptions, enableTele bool) bool {
teleFS := fs.NewTeleFS(t.logger)
tele := telemetry.NewTelemetry(enableTele, false, teleFS, t.logger, "", nil)
reportStorage := yaml.NewTestReportFS(t.logger)
mockStorage := yaml.NewYamlStore(path+"/tests", path, "", "", t.logger, tele)
return t.Test(path, testReportPath, appCmd, options, tele, reportStorage, mockStorage)
return t.Test(path, testReportPath, generateTestReport, appCmd, options, tele, reportStorage, mockStorage)
}

func (t *tester) InitialiseRunTestSet(cfg *RunTestSetConfig) InitialiseRunTestSetReturn {
Expand Down Expand Up @@ -357,7 +358,7 @@ func (t *tester) InitialiseRunTestSet(cfg *RunTestSetConfig) InitialiseRunTestSe
}
sortedConfigMocks := SortMocks(&fakeTestCase, readConfigMocks, t.logger)
t.logger.Info(fmt.Sprintf("the oss config mocks for %s are: %v\n", cfg.TestSet, readConfigMocks))

cfg.LoadedHooks.SetConfigMocks(sortedConfigMocks)
sort.SliceStable(readTcsMocks, func(i, j int) bool {
return readTcsMocks[i].Spec.ReqTimestampMock.Before(readTcsMocks[j].Spec.ReqTimestampMock)
Expand Down Expand Up @@ -398,11 +399,16 @@ func (t *tester) InitialiseRunTestSet(cfg *RunTestSetConfig) InitialiseRunTestSe
}

// starts the testrun
err = cfg.TestReportFS.Write(context.Background(), cfg.TestReportPath, returnVal.TestReport)
if err != nil {
t.logger.Error(err.Error())
returnVal.InitialStatus = models.TestRunStatusFailed
return returnVal
if cfg.GenerateTestReport {
err = cfg.TestReportFS.Write(context.Background(), cfg.TestReportPath, returnVal.TestReport)
if err != nil {
t.logger.Error(err.Error())
returnVal.InitialStatus = models.TestRunStatusFailed
return returnVal
}
} else {
index := strings.Split(cfg.TestSet, "-")[2]
returnVal.TestReport.Name = fmt.Sprintf("report-%v", index)
}

//if running keploy-tests along with unit tests
Expand Down Expand Up @@ -537,9 +543,10 @@ func (t *tester) FetchTestResults(cfg *FetchTestResultsConfig) models.TestRunSta
(*resultForTele)[0] += *cfg.Success
(*resultForTele)[1] += *cfg.Failure

err = cfg.TestReportFS.Write(context.Background(), cfg.TestReportPath, cfg.TestReport)

t.logger.Info("test report for "+cfg.TestSet+": ", zap.Any("name: ", cfg.TestReport.Name), zap.Any("path: ", cfg.Path+"/"+cfg.TestReport.Name))
if cfg.GenerateTestReport {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we are not generating the testreport we should atleast log once that we are skipping the test report since the flag has been set to false. Do this in all the services with this if condition

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

err = cfg.TestReportFS.Write(context.Background(), cfg.TestReportPath, cfg.TestReport)
t.logger.Info("test report for "+cfg.TestSet+": ", zap.Any("name: ", cfg.TestReport.Name), zap.Any("path: ", cfg.Path+"/"+cfg.TestReport.Name))
}

if *cfg.Status == models.TestRunStatusFailed {
pp.SetColorScheme(models.FailingColorScheme)
Expand All @@ -559,25 +566,26 @@ func (t *tester) FetchTestResults(cfg *FetchTestResultsConfig) models.TestRunSta
return *cfg.Status
}

// testSet, path, testReportPath, appCmd, appContainer, appNetwork, delay, pid, ys, loadedHooks, testReportFS, testRunChan, apiTimeout, ctx
func (t *tester) RunTestSet(testSet, path, testReportPath, appCmd, appContainer, appNetwork string, delay uint64, buildDelay time.Duration, pid uint32, testRunChan chan string, apiTimeout uint64, testcases map[string]bool, noiseConfig models.GlobalNoise, serveTest bool, initialisedValues TestEnvironmentSetup) models.TestRunStatus {
// testSet, path, testReportPath, generateTestReport, appCmd, appContainer, appNetwork, delay, pid, ys, loadedHooks, testReportFS, testRunChan, apiTimeout, ctx
func (t *tester) RunTestSet(testSet, path, testReportPath string, generateTestReport bool, appCmd, appContainer, appNetwork string, delay uint64, buildDelay time.Duration, pid uint32, testRunChan chan string, apiTimeout uint64, testcases map[string]bool, noiseConfig models.GlobalNoise, serveTest bool, initialisedValues TestEnvironmentSetup) models.TestRunStatus {
cfg := &RunTestSetConfig{
TestSet: testSet,
Path: path,
TestReportPath: testReportPath,
AppCmd: appCmd,
AppContainer: appContainer,
AppNetwork: appNetwork,
Delay: delay,
BuildDelay: buildDelay,
Pid: pid,
Storage: initialisedValues.Storage,
LoadedHooks: initialisedValues.LoadedHooks,
TestReportFS: initialisedValues.TestReportFS,
TestRunChan: testRunChan,
ApiTimeout: apiTimeout,
Ctx: initialisedValues.Ctx,
ServeTest: serveTest,
TestSet: testSet,
Path: path,
TestReportPath: testReportPath,
GenerateTestReport: generateTestReport,
AppCmd: appCmd,
AppContainer: appContainer,
AppNetwork: appNetwork,
Delay: delay,
BuildDelay: buildDelay,
Pid: pid,
Storage: initialisedValues.Storage,
LoadedHooks: initialisedValues.LoadedHooks,
TestReportFS: initialisedValues.TestReportFS,
TestRunChan: testRunChan,
ApiTimeout: apiTimeout,
Ctx: initialisedValues.Ctx,
ServeTest: serveTest,
}

initialisedTestSets := t.InitialiseRunTestSet(cfg)
Expand Down Expand Up @@ -707,15 +715,16 @@ func (t *tester) RunTestSet(testSet, path, testReportPath, appCmd, appContainer,
t.logger.Warn("These testcases have not been recorded by Keploy, may not work properly with Keploy.", zap.Strings("non-keploy mocks:", nonKeployTcs))
}
resultsCfg := &FetchTestResultsConfig{
TestReportFS: initialisedValues.TestReportFS,
TestReport: initialisedTestSets.TestReport,
Status: &status,
TestSet: testSet,
Success: &success,
Failure: &failure,
Ctx: initialisedValues.Ctx,
TestReportPath: testReportPath,
Path: path,
TestReportFS: initialisedValues.TestReportFS,
TestReport: initialisedTestSets.TestReport,
Status: &status,
TestSet: testSet,
Success: &success,
Failure: &failure,
Ctx: initialisedValues.Ctx,
TestReportPath: testReportPath,
GenerateTestReport: generateTestReport,
Path: path,
}
status = t.FetchTestResults(resultsCfg)
return status
Expand Down
Loading
Loading