From 3dbf599e445816969d30959e94618b910f6ccc60 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Wed, 17 Jan 2024 12:26:39 +0000 Subject: [PATCH 01/20] feat:support of pass throught hosts --- cmd/record.go | 13 +++-- cmd/test.go | 12 +++-- pkg/hooks/loader.go | 8 ++++ pkg/models/config.go | 48 ++++++++++--------- .../integrations/httpparser/httpparser.go | 13 +++-- pkg/service/generateConfig/generateConfig.go | 31 +++++++++++- pkg/service/mockrecord/mockrecord.go | 2 +- pkg/service/record/record.go | 3 +- pkg/service/record/service.go | 2 +- pkg/service/test/test.go | 3 ++ pkg/service/test/util.go | 1 + 11 files changed, 97 insertions(+), 39 deletions(-) diff --git a/cmd/record.go b/cmd/record.go index 35d7dd764..0ab559c2e 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -42,7 +42,7 @@ func readRecordConfig(configPath string) (*models.Record, error) { var filters = models.Filters{} -func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, configPath string) error { +func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, PassThroughHosts *[]string, configPath string) error { configFilePath := filepath.Join(configPath, "keploy-config.yaml") if isExist := utils.CheckFileExists(configFilePath); !isExist { return errFileNotFound @@ -79,6 +79,9 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string if len(*passThroughPorts) == 0 { *passThroughPorts = confRecord.PassThroughPorts } + if len(*PassThroughHosts) == 0 { + *PassThroughHosts = confRecord.BypassEndpointsRegistry + } return nil } @@ -156,10 +159,12 @@ func (r *Record) GetCmd() *cobra.Command { return err } - err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, configPath) + passThroughHosts := []string{} + + err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, &passThroughHosts, configPath) if err != nil { if err == errFileNotFound { - r.logger.Info("continuing without configuration file because file not found") + r.logger.Info("Keploy config not found, using default config") } else { r.logger.Error("", zap.Error(err)) } @@ -257,7 +262,7 @@ func (r *Record) GetCmd() *cobra.Command { } r.logger.Debug("the ports are", zap.Any("ports", ports)) - r.recorder.CaptureTraffic(path, proxyPort, appCmd, appContainer, networkName, delay, buildDelay, ports, &filters, enableTele) + r.recorder.CaptureTraffic(path, proxyPort, appCmd, appContainer, networkName, delay, buildDelay, ports, &filters, enableTele, passThroughHosts) return nil }, } diff --git a/cmd/test.go b/cmd/test.go index bfb028db7..1658934e2 100755 --- a/cmd/test.go +++ b/cmd/test.go @@ -42,7 +42,7 @@ func ReadTestConfig(configPath string) (*models.Test, error) { return &doc.Test, nil } -func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThorughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool) error { +func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThorughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool, passThroughHosts *[]string) error { configFilePath := filepath.Join(configPath, "keploy-config.yaml") if isExist := utils.CheckFileExists(configFilePath); !isExist { return errFileNotFound @@ -92,6 +92,9 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te if !*ignoreOrdering { *ignoreOrdering = confTest.IgnoreOrdering } + if len(*passThroughHosts) == 0 { + *passThroughHosts = confTest.BypassEndpointsRegistry + } return nil } @@ -241,10 +244,12 @@ func (t *Test) GetCmd() *cobra.Command { globalNoise := make(models.GlobalNoise) testsetNoise := make(models.TestsetNoise) - err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering) + passThroughHosts := []string{} + + err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering, &passThroughHosts) if err != nil { if err == errFileNotFound { - t.logger.Info("continuing without configuration file because file not found") + t.logger.Info("Keploy config not found, using default config") } else { t.logger.Error("", zap.Error(err)) } @@ -389,6 +394,7 @@ func (t *Test) GetCmd() *cobra.Command { WithCoverage: withCoverage, CoverageReportPath: coverageReportPath, IgnoreOrdering: ignoreOrdering, + PassthroughHosts: passThroughHosts, }, enableTele) } diff --git a/pkg/hooks/loader.go b/pkg/hooks/loader.go index 3b28885ce..52ca9c23e 100755 --- a/pkg/hooks/loader.go +++ b/pkg/hooks/loader.go @@ -141,6 +141,14 @@ func (h *Hook) GetProxyPort() uint32 { return h.proxyPort } +func (h *Hook) GetProxyHost() []string { + return h.passThroughHosts +} + +func (h *Hook) SetProxyHosts(passThroughHosts []string) { + h.passThroughHosts = passThroughHosts +} + func (h *Hook) AppendMocks(m *models.Mock, ctx context.Context) error { h.mu.Lock() defer h.mu.Unlock() diff --git a/pkg/models/config.go b/pkg/models/config.go index 2705101a9..3cedf8450 100644 --- a/pkg/models/config.go +++ b/pkg/models/config.go @@ -8,15 +8,16 @@ type Config struct { } type Record struct { - Path string `json:"path" yaml:"path"` - Command string `json:"command" yaml:"command"` - ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` - ContainerName string `json:"containerName" yaml:"containerName"` - NetworkName string `json:"networkName" yaml:"networkName"` - Delay uint64 `json:"delay" yaml:"delay"` - BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` - PassThroughPorts []uint `json:"passThroughPorts" yaml:"passThroughPorts"` - Filters Filters `json:"filters" yaml:"filters"` + Path string `json:"path" yaml:"path"` + Command string `json:"command" yaml:"command"` + ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` + ContainerName string `json:"containerName" yaml:"containerName"` + NetworkName string `json:"networkName" yaml:"networkName"` + Delay uint64 `json:"delay" yaml:"delay"` + BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` + PassThroughPorts []uint `json:"passThroughPorts" yaml:"passThroughPorts"` + BypassEndpointsRegistry []string `json:"bypassEndpointsRegistry" yaml:"bypassEndpointsRegistry"` + Filters Filters `json:"filters" yaml:"filters"` } type Filters struct { @@ -29,20 +30,21 @@ func (filter *Filters) GetKind() string { } type Test struct { - Path string `json:"path" yaml:"path"` - Command string `json:"command" yaml:"command"` - ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` - ContainerName string `json:"containerName" yaml:"containerName"` - NetworkName string `json:"networkName" yaml:"networkName"` - Tests map[string][]string `json:"tests" yaml:"tests"` - GlobalNoise Globalnoise `json:"globalNoise" yaml:"globalNoise"` - Delay uint64 `json:"delay" yaml:"delay"` - BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` - ApiTimeout uint64 `json:"apiTimeout" yaml:"apiTimeout"` - PassThroughPorts []uint `json:"passThroughPorts" yaml:"passThroughPorts"` - WithCoverage bool `json:"withCoverage" yaml:"withCoverage"` // boolean to capture the coverage in test - CoverageReportPath string `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files - IgnoreOrdering bool `json:"ignoreOrdering" yaml:"ignoreOrdering"` + Path string `json:"path" yaml:"path"` + Command string `json:"command" yaml:"command"` + ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` + ContainerName string `json:"containerName" yaml:"containerName"` + NetworkName string `json:"networkName" yaml:"networkName"` + Tests map[string][]string `json:"tests" yaml:"tests"` + GlobalNoise Globalnoise `json:"globalNoise" yaml:"globalNoise"` + Delay uint64 `json:"delay" yaml:"delay"` + BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` + ApiTimeout uint64 `json:"apiTimeout" yaml:"apiTimeout"` + PassThroughPorts []uint `json:"passThroughPorts" yaml:"passThroughPorts"` + BypassEndpointsRegistry []string `json:"bypassEndpointsRegistry" yaml:"bypassEndpointsRegistry"` + WithCoverage bool `json:"withCoverage" yaml:"withCoverage"` // boolean to capture the coverage in test + CoverageReportPath string `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files + IgnoreOrdering bool `json:"ignoreOrdering" yaml:"ignoreOrdering"` } type Globalnoise struct { diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index 4d4923d46..1655134c2 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -489,11 +489,12 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * if err != nil { logger.Error("error while matching http mocks", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) } - if !isMatched { passthroughHost := false - for _, host := range models.PassThroughHosts { - if req.Host == host { + passthroughHosts := models.PassThroughHosts + passthroughHosts = append(passthroughHosts, h.GetProxyHost()...) + for _, host := range passthroughHosts { + if req.Host == host || req.URL.String() == host { passthroughHost = true } } @@ -796,8 +797,10 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime "operation": req.Method, } passthroughHost := false - for _, host := range models.PassThroughHosts { - if req.Host == host { + passthroughHosts := models.PassThroughHosts + passthroughHosts = append(passthroughHosts, h.GetProxyHost()...) + for _, host := range passthroughHosts { + if req.Host == host || req.URL.String() == host { passthroughHost = true } } diff --git a/pkg/service/generateConfig/generateConfig.go b/pkg/service/generateConfig/generateConfig.go index 09752c595..e401e728f 100644 --- a/pkg/service/generateConfig/generateConfig.go +++ b/pkg/service/generateConfig/generateConfig.go @@ -35,6 +35,7 @@ record: delay: 5 buildDelay: 30s passThroughPorts: [] + bypassEndpointsRegistry: [] filters: ReqHeader: [] urlMethods: {} @@ -57,9 +58,37 @@ test: apiTimeout: 5 ignoreOrdering: false passThroughPorts: [] + bypassEndpointsRegistry: [] withCoverage: false coverageReportPath: "" - ` + # + # Example on using globalNoise + # bypassEndpointsRegistry: ["api/users"] + # globalNoise: + # global: + # body: { + # # to ignore some values for a field, + # # pass regex patterns to the corresponding array value + # "url": ["https?://\S+", "http://\S+"], + # } + # header: { + # # to ignore the entire field, pass an empty array + # "Date: [], + # } + # # to ignore fields or the corresponding values for a specific test-set, + # # pass the test-set-name as a key to the "test-sets" object and + # # populate the corresponding "body" and "header" objects + # test-sets: + # test-set-1: + # body: { + # # ignore all the values for the "url" field + # "url": [] + # } + # header: { + # # we can also pass the exact value to ignore for a field + # "User-Agent": ["PostmanRuntime/7.34.0"] + # } +` func (g *generatorConfig) GenerateConfig(filePath string) { var node yaml.Node diff --git a/pkg/service/mockrecord/mockrecord.go b/pkg/service/mockrecord/mockrecord.go index 4b9d8b87e..260a05da5 100644 --- a/pkg/service/mockrecord/mockrecord.go +++ b/pkg/service/mockrecord/mockrecord.go @@ -51,7 +51,7 @@ func (s *mockRecorder) MockRecord(path string, proxyPort uint32, pid uint32, moc s.logger.Error("error while creating hooks", zap.Error(err)) return } - + if err := loadedHooks.LoadHooks("", "", pid, ctx, nil); err != nil { return } diff --git a/pkg/service/record/record.go b/pkg/service/record/record.go index f72566acc..67c645bac 100755 --- a/pkg/service/record/record.go +++ b/pkg/service/record/record.go @@ -29,7 +29,7 @@ func NewRecorder(logger *zap.Logger) Recorder { } } -func (r *recorder) CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, appNetwork string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.Filters, enableTele bool) { +func (r *recorder) CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, appNetwork string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.Filters, enableTele bool, passThroughHosts []string) { var ps *proxy.ProxySet stopper := make(chan os.Signal, 1) @@ -50,6 +50,7 @@ func (r *recorder) CaptureTraffic(path string, proxyPort uint32, appCmd, appCont routineId := pkg.GenerateRandomID() // Initiate the hooks and update the vaccant ProxyPorts map loadedHooks, err := hooks.NewHook(ys, routineId, r.Logger) + loadedHooks.SetProxyHosts(passThroughHosts) if err != nil { r.Logger.Error("error while creating hooks", zap.Error(err)) return diff --git a/pkg/service/record/service.go b/pkg/service/record/service.go index 3b28b21bb..7009d06e5 100755 --- a/pkg/service/record/service.go +++ b/pkg/service/record/service.go @@ -7,5 +7,5 @@ import ( ) type Recorder interface { - CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, networkName string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.Filters, enableTele bool) + CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, networkName string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.Filters, enableTele bool, passThroughHosts []string) } diff --git a/pkg/service/test/test.go b/pkg/service/test/test.go index 9ad6e4ac2..e3148a04e 100755 --- a/pkg/service/test/test.go +++ b/pkg/service/test/test.go @@ -52,6 +52,7 @@ type TestOptions struct { WithCoverage bool CoverageReportPath string IgnoreOrdering bool + PassthroughHosts []string } func NewTester(logger *zap.Logger) Tester { @@ -124,6 +125,7 @@ func (t *tester) InitialiseTest(cfg *TestConfig) (InitialiseTestReturn, error) { routineId := pkg.GenerateRandomID() // Initiate the hooks returnVal.LoadedHooks, err = hooks.NewHook(returnVal.YamlStore, routineId, t.logger) + returnVal.LoadedHooks.SetProxyHosts(cfg.PassThroughHosts) if err != nil { return returnVal, fmt.Errorf("error while creating hooks %v", err) } @@ -217,6 +219,7 @@ func (t *tester) Test(path string, testReportPath string, appCmd string, options WithCoverage: options.WithCoverage, CoverageReportPath: options.CoverageReportPath, EnableTele: enableTele, + PassThroughHosts: options.PassthroughHosts, } initialisedValues, err := t.InitialiseTest(cfg) // Recover from panic and gracefully shutdown diff --git a/pkg/service/test/util.go b/pkg/service/test/util.go index 7d571c9da..63e62266b 100755 --- a/pkg/service/test/util.go +++ b/pkg/service/test/util.go @@ -59,6 +59,7 @@ type TestConfig struct { WithCoverage bool CoverageReportPath string EnableTele bool + PassThroughHosts []string } type RunTestSetConfig struct { From 758225757f2319bee257fa18d3e8d66bcfca4496 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Thu, 18 Jan 2024 12:15:43 +0000 Subject: [PATCH 02/20] refactor:filter, stub and passthrough --- cmd/record.go | 28 +++++--- cmd/test.go | 22 ++++--- pkg/hooks/connection/factory.go | 4 +- pkg/hooks/loader.go | 22 +++++-- pkg/models/config.go | 37 +++++++---- pkg/platform/yaml/yaml.go | 62 ++++++++++++------ .../integrations/httpparser/httpparser.go | 64 ++++++++++++++++--- pkg/proxy/integrations/scram/util.go | 2 +- pkg/proxy/proxy.go | 2 + pkg/service/generateConfig/generateConfig.go | 53 ++++++++++++--- pkg/service/record/record.go | 2 +- pkg/service/record/service.go | 2 +- pkg/service/test/test.go | 2 +- pkg/service/test/util.go | 2 +- 14 files changed, 221 insertions(+), 83 deletions(-) diff --git a/cmd/record.go b/cmd/record.go index 0ab559c2e..ada7a6954 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -40,9 +40,9 @@ func readRecordConfig(configPath string) (*models.Record, error) { return &doc.Record, nil } -var filters = models.Filters{} +var filters = models.TestFilter{} -func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, PassThroughHosts *[]string, configPath string) error { +func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, passThrough *[]models.Filters, configPath string) error { configFilePath := filepath.Join(configPath, "keploy-config.yaml") if isExist := utils.CheckFileExists(configFilePath); !isExist { return errFileNotFound @@ -57,7 +57,9 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string if len(*path) == 0 { *path = confRecord.Path } - filters = confRecord.Filters + + filters = confRecord.Tests + if *proxyPort == 0 { *proxyPort = confRecord.ProxyPort } @@ -76,12 +78,18 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string if *buildDelay == 30*time.Second && confRecord.BuildDelay != 0 { *buildDelay = confRecord.BuildDelay } + *passThrough = append(*passThrough, confRecord.Stubs.Filters...) + if len(*passThroughPorts) == 0 { - *passThroughPorts = confRecord.PassThroughPorts - } - if len(*PassThroughHosts) == 0 { - *PassThroughHosts = confRecord.BypassEndpointsRegistry + for _, filter := range confRecord.Stubs.Filters { + if filter.Port != 0 && filter.Host == "" && filter.Path == "" { + *passThroughPorts = append(*passThroughPorts, filter.Port) + } else { + *passThrough = append(*passThrough, filter) + } + } } + return nil } @@ -159,9 +167,9 @@ func (r *Record) GetCmd() *cobra.Command { return err } - passThroughHosts := []string{} + passThrough := []models.Filters{} - err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, &passThroughHosts, configPath) + err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, &passThrough, configPath) if err != nil { if err == errFileNotFound { r.logger.Info("Keploy config not found, using default config") @@ -262,7 +270,7 @@ func (r *Record) GetCmd() *cobra.Command { } r.logger.Debug("the ports are", zap.Any("ports", ports)) - r.recorder.CaptureTraffic(path, proxyPort, appCmd, appContainer, networkName, delay, buildDelay, ports, &filters, enableTele, passThroughHosts) + r.recorder.CaptureTraffic(path, proxyPort, appCmd, appContainer, networkName, delay, buildDelay, ports, &filters, enableTele, passThrough) return nil }, } diff --git a/cmd/test.go b/cmd/test.go index 1658934e2..b126e0fad 100755 --- a/cmd/test.go +++ b/cmd/test.go @@ -42,7 +42,7 @@ func ReadTestConfig(configPath string) (*models.Test, error) { return &doc.Test, nil } -func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThorughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool, passThroughHosts *[]string) error { +func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool, passThroughHosts *[]models.Filters) error { configFilePath := filepath.Join(configPath, "keploy-config.yaml") if isExist := utils.CheckFileExists(configFilePath); !isExist { return errFileNotFound @@ -60,7 +60,7 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te if *appCmd == "" { *appCmd = confTest.Command } - for testset, testcases := range confTest.Tests { + for testset, testcases := range confTest.SelectedTests { if _, ok := (*tests)[testset]; !ok { (*tests)[testset] = testcases } @@ -77,9 +77,7 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te if *buildDelay == 30*time.Second && confTest.BuildDelay != 0 { *buildDelay = confTest.BuildDelay } - if len(*passThorughPorts) == 0 { - *passThorughPorts = confTest.PassThroughPorts - } + if len(*coverageReportPath) == 0 { *coverageReportPath = confTest.CoverageReportPath } @@ -92,9 +90,17 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te if !*ignoreOrdering { *ignoreOrdering = confTest.IgnoreOrdering } - if len(*passThroughHosts) == 0 { - *passThroughHosts = confTest.BypassEndpointsRegistry + *passThroughHosts = append(*passThroughHosts, confTest.Stubs.Filters...) + if len(*passThroughPorts) == 0 { + for _, filter := range confTest.Stubs.Filters { + if filter.Port != 0 && filter.Host == "" && filter.Path == "" { + *passThroughPorts = append(*passThroughPorts, filter.Port) + } else { + *passThroughHosts = append(*passThroughHosts, filter) + } + } } + return nil } @@ -244,7 +250,7 @@ func (t *Test) GetCmd() *cobra.Command { globalNoise := make(models.GlobalNoise) testsetNoise := make(models.TestsetNoise) - passThroughHosts := []string{} + passThroughHosts := []models.Filters{} err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering, &passThroughHosts) if err != nil { diff --git a/pkg/hooks/connection/factory.go b/pkg/hooks/connection/factory.go index 2b79717b0..f0872a668 100755 --- a/pkg/hooks/connection/factory.go +++ b/pkg/hooks/connection/factory.go @@ -38,7 +38,7 @@ func NewFactory(inactivityThreshold time.Duration, logger *zap.Logger) *Factory // ProcessActiveTrackers iterates over all connection the trackers and checks if they are complete. If so, it captures the ingress call and // deletes the tracker. If the tracker is inactive for a long time, it deletes it. -func (factory *Factory) ProcessActiveTrackers(db platform.TestCaseDB, ctx context.Context, filters *models.Filters) { +func (factory *Factory) ProcessActiveTrackers(db platform.TestCaseDB, ctx context.Context, filters *models.TestFilter) { factory.mutex.Lock() defer factory.mutex.Unlock() var trackersToDelete []structs.ConnID @@ -98,7 +98,7 @@ func (factory *Factory) GetOrCreate(connectionID structs.ConnID) *Tracker { return tracker } -func capture(db platform.TestCaseDB, req *http.Request, resp *http.Response, logger *zap.Logger, ctx context.Context, reqTimeTest time.Time, resTimeTest time.Time, filters *models.Filters) { +func capture(db platform.TestCaseDB, req *http.Request, resp *http.Response, logger *zap.Logger, ctx context.Context, reqTimeTest time.Time, resTimeTest time.Time, filters *models.TestFilter) { reqBody, err := io.ReadAll(req.Body) if err != nil { logger.Error("failed to read the http request body", zap.Error(err)) diff --git a/pkg/hooks/loader.go b/pkg/hooks/loader.go index 52ca9c23e..5cd3e122f 100755 --- a/pkg/hooks/loader.go +++ b/pkg/hooks/loader.go @@ -93,7 +93,9 @@ type Hook struct { writev link.Link writevRet link.Link - idc clients.InternalDockerClient + idc clients.InternalDockerClient + passThroughHosts models.Stubs + sourcePort int } func NewHook(db platform.TestCaseDB, mainRoutineId int, logger *zap.Logger) (*Hook, error) { @@ -141,12 +143,22 @@ func (h *Hook) GetProxyPort() uint32 { return h.proxyPort } -func (h *Hook) GetProxyHost() []string { +func (h *Hook) GetProxyHost() models.Stubs { return h.passThroughHosts } -func (h *Hook) SetProxyHosts(passThroughHosts []string) { - h.passThroughHosts = passThroughHosts +func (h *Hook) SetProxyHosts(passThroughHosts []models.Filters) { + h.passThroughHosts = models.Stubs{ + Filters: passThroughHosts, + } +} + +func (h *Hook) GetSourcePort() int { + return h.sourcePort +} + +func (h *Hook) SetSourcePort(sourcePort int) { + h.sourcePort = sourcePort } func (h *Hook) AppendMocks(m *models.Mock, ctx context.Context) error { @@ -517,7 +529,7 @@ func (h *Hook) Stop(forceStop bool) { // $BPF_CLANG and $BPF_CFLAGS are set by the Makefile. // //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -no-global-types -target $TARGET bpf keploy_ebpf.c -- -I./headers -I./headers/$TARGET -func (h *Hook) LoadHooks(appCmd, appContainer string, pid uint32, ctx context.Context, filters *models.Filters) error { +func (h *Hook) LoadHooks(appCmd, appContainer string, pid uint32, ctx context.Context, filters *models.TestFilter) error { if err := settings.InitRealTimeOffset(); err != nil { h.logger.Error("failed to fix the BPF clock", zap.Error(err)) return err diff --git a/pkg/models/config.go b/pkg/models/config.go index 3cedf8450..45ec57fc3 100644 --- a/pkg/models/config.go +++ b/pkg/models/config.go @@ -8,25 +8,34 @@ type Config struct { } type Record struct { - Path string `json:"path" yaml:"path"` - Command string `json:"command" yaml:"command"` - ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` - ContainerName string `json:"containerName" yaml:"containerName"` - NetworkName string `json:"networkName" yaml:"networkName"` - Delay uint64 `json:"delay" yaml:"delay"` - BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` - PassThroughPorts []uint `json:"passThroughPorts" yaml:"passThroughPorts"` - BypassEndpointsRegistry []string `json:"bypassEndpointsRegistry" yaml:"bypassEndpointsRegistry"` - Filters Filters `json:"filters" yaml:"filters"` + Path string `json:"path" yaml:"path"` + Command string `json:"command" yaml:"command"` + ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` + ContainerName string `json:"containerName" yaml:"containerName"` + NetworkName string `json:"networkName" yaml:"networkName"` + Delay uint64 `json:"delay" yaml:"delay"` + BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` + Tests TestFilter `json:"tests" yaml:"tests"` + Stubs Stubs `json:"stubs" yaml:"stubs"` } +type TestFilter struct { + Filters []Filters `json:"filters" yaml:"filters"` +} + +type Stubs struct { + Filters []Filters `json:"filters" yaml:"filters"` +} type Filters struct { - ReqHeader []string `json:"req_header" yaml:"req_header"` - URLMethods map[string][]string `json:"urlMethods" yaml:"urlMethods"` + Path string `json:"path" yaml:"path"` + UrlMethods []string `json:"urlMethods" yaml:"urlMethods"` + Host string `json:"host" yaml:"host"` + Headers map[string]string `json:"headers" yaml:"headers"` + Port uint `json:"ports" yaml:"ports"` } -func (filter *Filters) GetKind() string { - return "filter" +func (tests *TestFilter) GetKind() string { + return "Tests" } type Test struct { diff --git a/pkg/platform/yaml/yaml.go b/pkg/platform/yaml/yaml.go index 848a9d24b..44842146f 100755 --- a/pkg/platform/yaml/yaml.go +++ b/pkg/platform/yaml/yaml.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -129,33 +130,51 @@ func (ys *Yaml) Write(path, fileName string, docRead platform.KindSpecifier) err return nil } -func containsMatchingUrl(urlMethods map[string][]string, urlStr string, requestMethod models.Method) bool { - parsedURL, err := url.Parse(urlStr) +func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, requestMethod models.Method) bool { + urlMatched := false + parsedURL, err := url.Parse(requestUrl) if err != nil { return false } // Check for URL path and method - path := parsedURL.Path - if methods, exists := urlMethods[path]; exists { - // Loop through the methods for this path - for _, method := range methods { - // If the request method matches one of the allowed methods, return true + regex, err := regexp.Compile(urlStr) + if err != nil { + return false + } + + urlMatch := regex.FindStringSubmatch(parsedURL.Path) + + if len(urlMatch) > 0 && len(urlStr) != 0 { + urlMatched = true + } + + if len(urlMethods) != 0 { + urlMatched = false + for _, method := range urlMethods { if string(method) == string(requestMethod) { - return true + urlMatched = true } } - // If the request method is not in the allowed methods, return false - return false } - return false + return urlMatched } -func hasBannedHeaders(object map[string]string, bannedHeaders []string) bool { - for headerName, _ := range object { - for _, bannedHeader := range bannedHeaders { - if headerName == bannedHeader { +func hasBannedHeaders(object map[string]string, bannedHeaders map[string]string) bool { + for headerName, headerNameValue := range object { + for bannedHeaderName, bannedHeaderValue := range bannedHeaders { + regex, err := regexp.Compile(bannedHeaderValue) + if err != nil { + continue + } + headerNameMatch := regex.FindStringSubmatch(headerNameValue) + regex, err = regexp.Compile(bannedHeaderValue) + if err != nil { + continue + } + headerValueMatch := regex.FindStringSubmatch(headerNameValue) + if len(headerNameMatch) > 0 || len(headerValueMatch) > 0 || headerName == bannedHeaderName || bannedHeaderValue == headerNameValue { return true } } @@ -168,18 +187,19 @@ func (ys *Yaml) WriteTestcase(tcRead platform.KindSpecifier, ctx context.Context if !ok { return fmt.Errorf("%s failed to read testcase in WriteTestcase", Emoji) } - filters, ok := filtersRead.(*models.Filters) + testFilters, ok := filtersRead.(*models.TestFilter) var bypassTestCase = false if ok { - if containsMatchingUrl(filters.URLMethods, tc.HttpReq.URL, tc.HttpReq.Method) { - bypassTestCase = true - } else if hasBannedHeaders(tc.HttpReq.Header, filters.ReqHeader) { - bypassTestCase = true + for _, testFilter := range testFilters.Filters { + if containsMatchingUrl(testFilter.UrlMethods, testFilter.Path, tc.HttpReq.URL, tc.HttpReq.Method) { + bypassTestCase = true + } else if hasBannedHeaders(tc.HttpReq.Header, testFilter.Headers) { + bypassTestCase = true + } } } - if !bypassTestCase { if ys.tele != nil { ys.tele.RecordedTestAndMocks() diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index 1655134c2..ed90fb690 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -12,6 +12,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strconv" "strings" "time" @@ -491,10 +492,35 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * } if !isMatched { passthroughHost := false - passthroughHosts := models.PassThroughHosts - passthroughHosts = append(passthroughHosts, h.GetProxyHost()...) - for _, host := range passthroughHosts { - if req.Host == host || req.URL.String() == host { + passThrough := models.PassThroughHosts + portPassThrough := []models.Filters{} + for _, filters := range h.GetProxyHost().Filters { + if filters.Port == 0 { + passThrough = append(passThrough, filters.Host, filters.Path) + } else { + portPassThrough = append(portPassThrough, filters) + } + } + // Define the regular expression pattern + + for _, host := range passThrough { + regex, err := regexp.Compile(host) + if err != nil { + continue + } + submatches := regex.FindStringSubmatch(req.URL.String()) + if len(submatches) > 0 && host != "" || req.Host == host { + passthroughHost = true + } + + } + for _, filter := range portPassThrough { + regex, err := regexp.Compile(filter.Path) + if err != nil { + continue + } + submatches := regex.FindStringSubmatch(req.URL.String()) + if h.GetSourcePort() == int(filter.Port) && (len(submatches) > 0 && filter.Path != "") { passthroughHost = true } } @@ -797,10 +823,32 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime "operation": req.Method, } passthroughHost := false - passthroughHosts := models.PassThroughHosts - passthroughHosts = append(passthroughHosts, h.GetProxyHost()...) - for _, host := range passthroughHosts { - if req.Host == host || req.URL.String() == host { + passThrough := models.PassThroughHosts + portPassThrough := []models.Filters{} + for _, filters := range h.GetProxyHost().Filters { + if filters.Port == 0 { + passThrough = append(passThrough, filters.Host, filters.Path) + } else { + portPassThrough = append(portPassThrough, filters) + } + } + for _, host := range passThrough { + regex, err := regexp.Compile(host) + if err != nil { + continue + } + submatches := regex.FindStringSubmatch(req.URL.String()) + if len(submatches) > 0 && host != "" || req.Host == host { + passthroughHost = true + } + } + for _, filter := range portPassThrough { + regex, err := regexp.Compile(filter.Path) + if err != nil { + continue + } + submatches := regex.FindStringSubmatch(req.URL.String()) + if h.GetSourcePort() == int(filter.Port) && (len(submatches) > 0 && filter.Path != "") { passthroughHost = true } } diff --git a/pkg/proxy/integrations/scram/util.go b/pkg/proxy/integrations/scram/util.go index 2b6da92da..5260517cb 100644 --- a/pkg/proxy/integrations/scram/util.go +++ b/pkg/proxy/integrations/scram/util.go @@ -32,7 +32,7 @@ func extractClientNonce(firstMsg string) (string, error) { // value := strings.Split(part, "=")[1] value := strings.TrimPrefix(part, "r=") if value == part { - return "", fmt.Errorf("error parsing '%s' for fetching client nonce", part, ) + return "", fmt.Errorf("error parsing '%s' for fetching client nonce", part) } return value, nil } diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index 95e5bd7b7..c2793c988 100755 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -775,6 +775,8 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con remoteAddr := conn.RemoteAddr().(*net.TCPAddr) sourcePort := remoteAddr.Port + ps.hook.SetSourcePort(sourcePort) + ps.logger.Debug("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) //TODO: fix this bug, getting source port same as proxy port. diff --git a/pkg/service/generateConfig/generateConfig.go b/pkg/service/generateConfig/generateConfig.go index e401e728f..a85dce387 100644 --- a/pkg/service/generateConfig/generateConfig.go +++ b/pkg/service/generateConfig/generateConfig.go @@ -34,11 +34,17 @@ record: networkName: "" delay: 5 buildDelay: 30s - passThroughPorts: [] - bypassEndpointsRegistry: [] - filters: - ReqHeader: [] - urlMethods: {} + tests: + filters: + - path: "" + urlMethods: [] + headers: {} + host: "" + stubs: + filters: + - path: "" + host: "" + port: 0 test: path: "" # mandatory @@ -47,7 +53,7 @@ test: containerName: "" networkName: "" # example: "test-set-1": ["test-1", "test-2", "test-3"] - tests: + selectedTests: # to use globalNoise, please follow the guide at the end of this file. globalNoise: global: @@ -57,13 +63,40 @@ test: buildDelay: 30s apiTimeout: 5 ignoreOrdering: false - passThroughPorts: [] - bypassEndpointsRegistry: [] + tests: + filters: + - path: "" + urlMethods: [] + headers: {} + host: "" + stubs: + filters: + - path: "" + host: "" + port: 0 withCoverage: false coverageReportPath: "" + # Example on using tests + # tests: + # filters: + # - path: "/user/app" + # urlMethods: ["GET"] + # headers: { + # "^asdf*": "^test" + # } + # host: "dc.services.visualstudio.com" + # Example on using stubs + # stubs: + # filters: + # - path: "/user/app" + # port: 8080 + # - port: 8081 + # - host: "dc.services.visualstudio.com" + # - port: 8081 + # host: "dc.services.visualstudio.com" + # path: "/user/app" # # Example on using globalNoise - # bypassEndpointsRegistry: ["api/users"] # globalNoise: # global: # body: { @@ -73,7 +106,7 @@ test: # } # header: { # # to ignore the entire field, pass an empty array - # "Date: [], + # "Date": [], # } # # to ignore fields or the corresponding values for a specific test-set, # # pass the test-set-name as a key to the "test-sets" object and diff --git a/pkg/service/record/record.go b/pkg/service/record/record.go index 67c645bac..a71ae73b8 100755 --- a/pkg/service/record/record.go +++ b/pkg/service/record/record.go @@ -29,7 +29,7 @@ func NewRecorder(logger *zap.Logger) Recorder { } } -func (r *recorder) CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, appNetwork string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.Filters, enableTele bool, passThroughHosts []string) { +func (r *recorder) CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, appNetwork string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.TestFilter, enableTele bool, passThroughHosts []models.Filters) { var ps *proxy.ProxySet stopper := make(chan os.Signal, 1) diff --git a/pkg/service/record/service.go b/pkg/service/record/service.go index 7009d06e5..35efd91af 100755 --- a/pkg/service/record/service.go +++ b/pkg/service/record/service.go @@ -7,5 +7,5 @@ import ( ) type Recorder interface { - CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, networkName string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.Filters, enableTele bool, passThroughHosts []string) + CaptureTraffic(path string, proxyPort uint32, appCmd, appContainer, networkName string, Delay uint64, buildDelay time.Duration, ports []uint, filters *models.TestFilter, enableTele bool, passThroughHosts []models.Filters) } diff --git a/pkg/service/test/test.go b/pkg/service/test/test.go index e3148a04e..ed65ee418 100755 --- a/pkg/service/test/test.go +++ b/pkg/service/test/test.go @@ -52,7 +52,7 @@ type TestOptions struct { WithCoverage bool CoverageReportPath string IgnoreOrdering bool - PassthroughHosts []string + PassthroughHosts []models.Filters } func NewTester(logger *zap.Logger) Tester { diff --git a/pkg/service/test/util.go b/pkg/service/test/util.go index 63e62266b..e2c4460cf 100755 --- a/pkg/service/test/util.go +++ b/pkg/service/test/util.go @@ -59,7 +59,7 @@ type TestConfig struct { WithCoverage bool CoverageReportPath string EnableTele bool - PassThroughHosts []string + PassThroughHosts []models.Filters } type RunTestSetConfig struct { From 9d21c523d8001bb376a8cac43ba2184aacaddef2 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Fri, 19 Jan 2024 10:27:22 +0000 Subject: [PATCH 03/20] refactor:passthroughthosts --- pkg/models/const.go | 2 +- pkg/proxy/integrations/httpparser/httpparser.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/models/const.go b/pkg/models/const.go index 4cb2a0b4d..d442d3e99 100755 --- a/pkg/models/const.go +++ b/pkg/models/const.go @@ -16,7 +16,7 @@ const ( ) var ( - PassThroughHosts = []string{"dc.services.visualstudio.com"} + PassThroughHosts = []string{"dc\\.services\\.visualstudio\\.com"} ) var orangeColorSGR = []color.Attribute{38, 5, 208} diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index ed90fb690..6b638e613 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -508,8 +508,8 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * if err != nil { continue } - submatches := regex.FindStringSubmatch(req.URL.String()) - if len(submatches) > 0 && host != "" || req.Host == host { + subMatches := regex.FindStringSubmatch(req.URL.String()) + if len(subMatches) > 0 && host != "" || req.Host == host { passthroughHost = true } @@ -519,8 +519,8 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * if err != nil { continue } - submatches := regex.FindStringSubmatch(req.URL.String()) - if h.GetSourcePort() == int(filter.Port) && (len(submatches) > 0 && filter.Path != "") { + subMatches := regex.FindStringSubmatch(req.URL.String()) + if h.GetSourcePort() == int(filter.Port) && (len(subMatches) > 0 && filter.Path != "") { passthroughHost = true } } @@ -837,8 +837,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime if err != nil { continue } - submatches := regex.FindStringSubmatch(req.URL.String()) - if len(submatches) > 0 && host != "" || req.Host == host { + subMatches := regex.FindStringSubmatch(req.URL.String()) + if len(subMatches) > 0 && host != "" || req.Host == host { passthroughHost = true } } @@ -847,8 +847,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime if err != nil { continue } - submatches := regex.FindStringSubmatch(req.URL.String()) - if h.GetSourcePort() == int(filter.Port) && (len(submatches) > 0 && filter.Path != "") { + subMatches := regex.FindStringSubmatch(req.URL.String()) + if h.GetSourcePort() == int(filter.Port) && (len(subMatches) > 0 && filter.Path != "") { passthroughHost = true } } From 4f34372c2ddcff1cd5eafa0cfc12c5ef392e7a93 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Fri, 19 Jan 2024 10:35:41 +0000 Subject: [PATCH 04/20] refactor: pipeline code --- .github/workflows/test_workflow_scripts/node-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_workflow_scripts/node-linux.sh b/.github/workflows/test_workflow_scripts/node-linux.sh index f9cb2417f..26380ce1a 100755 --- a/.github/workflows/test_workflow_scripts/node-linux.sh +++ b/.github/workflows/test_workflow_scripts/node-linux.sh @@ -74,7 +74,7 @@ sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 --testset # Update the global noise to ts. config_file="./keploy-config.yaml" -sed -i '/tests:/a \ "test-set-0": ["test-1", "test-2"]' "$config_file" +sed -i '/selectedTests:/a \ "test-set-0": ["test-1", "test-2"]' "$config_file" sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --apiTimeout 30 --delay 10 From 0f249005fabb857c3fb202f481e997d54df09b7c Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Fri, 19 Jan 2024 10:39:39 +0000 Subject: [PATCH 05/20] refactor:PassThroughHosts --- pkg/models/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/const.go b/pkg/models/const.go index d442d3e99..c40e7aa2f 100755 --- a/pkg/models/const.go +++ b/pkg/models/const.go @@ -16,7 +16,7 @@ const ( ) var ( - PassThroughHosts = []string{"dc\\.services\\.visualstudio\\.com"} + PassThroughHosts = []string{"^dc\\.services\\.visualstudio\\.com$"} ) var orangeColorSGR = []color.Attribute{38, 5, 208} From 8c1d13867dce3c7e16423206a0482f990374cd91 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Wed, 17 Jan 2024 12:26:39 +0000 Subject: [PATCH 06/20] feat:support of pass throught hosts --- cmd/record.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/record.go b/cmd/record.go index ada7a6954..7fa09bd4e 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -89,7 +89,6 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string } } } - return nil } From 477870ded89a83367082500d3cf310d279b19dbb Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Thu, 18 Jan 2024 12:15:43 +0000 Subject: [PATCH 07/20] refactor:filter, stub and passthrough --- cmd/record.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/record.go b/cmd/record.go index 7fa09bd4e..ada7a6954 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -89,6 +89,7 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string } } } + return nil } From 06b1aee6450645bfc0e27e743855d5a0ef1a6d40 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 22 Jan 2024 10:04:18 +0000 Subject: [PATCH 08/20] refactor:parse and loader --- cmd/record.go | 14 ++-- pkg/hooks/loader.go | 9 --- pkg/platform/yaml/yaml.go | 39 ++++++----- .../integrations/grpcparser/grpc_parser.go | 2 +- .../integrations/httpparser/httpparser.go | 38 +++++----- .../integrations/mongoparser/mongoparser.go | 2 +- .../integrations/mysqlparser/mysqlparser.go | 2 +- .../postgresParser/postgres_parser.go | 2 +- pkg/proxy/proxy.go | 8 +-- pkg/service/generateConfig/generateConfig.go | 45 ------------ utils/utils.go | 70 ++++++++++++------- 11 files changed, 99 insertions(+), 132 deletions(-) diff --git a/cmd/record.go b/cmd/record.go index ada7a6954..586b6e2f4 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -78,15 +78,13 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string if *buildDelay == 30*time.Second && confRecord.BuildDelay != 0 { *buildDelay = confRecord.BuildDelay } - *passThrough = append(*passThrough, confRecord.Stubs.Filters...) + passThroughPortProvided := len(*passThroughPorts) == 0 - if len(*passThroughPorts) == 0 { - for _, filter := range confRecord.Stubs.Filters { - if filter.Port != 0 && filter.Host == "" && filter.Path == "" { - *passThroughPorts = append(*passThroughPorts, filter.Port) - } else { - *passThrough = append(*passThrough, filter) - } + for _, filter := range confRecord.Stubs.Filters { + if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided { + *passThroughPorts = append(*passThroughPorts, filter.Port) + } else { + *passThrough = append(*passThrough, filter) } } diff --git a/pkg/hooks/loader.go b/pkg/hooks/loader.go index 5cd3e122f..19d7727c0 100755 --- a/pkg/hooks/loader.go +++ b/pkg/hooks/loader.go @@ -95,7 +95,6 @@ type Hook struct { idc clients.InternalDockerClient passThroughHosts models.Stubs - sourcePort int } func NewHook(db platform.TestCaseDB, mainRoutineId int, logger *zap.Logger) (*Hook, error) { @@ -153,14 +152,6 @@ func (h *Hook) SetProxyHosts(passThroughHosts []models.Filters) { } } -func (h *Hook) GetSourcePort() int { - return h.sourcePort -} - -func (h *Hook) SetSourcePort(sourcePort int) { - h.sourcePort = sourcePort -} - func (h *Hook) AppendMocks(m *models.Mock, ctx context.Context) error { h.mu.Lock() defer h.mu.Unlock() diff --git a/pkg/platform/yaml/yaml.go b/pkg/platform/yaml/yaml.go index 44842146f..2f507c399 100755 --- a/pkg/platform/yaml/yaml.go +++ b/pkg/platform/yaml/yaml.go @@ -130,22 +130,22 @@ func (ys *Yaml) Write(path, fileName string, docRead platform.KindSpecifier) err return nil } -func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, requestMethod models.Method) bool { +func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, requestMethod models.Method) (error, bool) { urlMatched := false parsedURL, err := url.Parse(requestUrl) if err != nil { - return false + return err, false } // Check for URL path and method regex, err := regexp.Compile(urlStr) if err != nil { - return false + return err, false } - urlMatch := regex.FindStringSubmatch(parsedURL.Path) + urlMatch := regex.MatchString(parsedURL.Path) - if len(urlMatch) > 0 && len(urlStr) != 0 { + if urlMatch && len(urlStr) != 0 { urlMatched = true } @@ -158,28 +158,29 @@ func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, } } - return urlMatched + return nil, urlMatched } -func hasBannedHeaders(object map[string]string, bannedHeaders map[string]string) bool { +func hasBannedHeaders(object map[string]string, bannedHeaders map[string]string) (error, bool) { for headerName, headerNameValue := range object { for bannedHeaderName, bannedHeaderValue := range bannedHeaders { - regex, err := regexp.Compile(bannedHeaderValue) + regex, err := regexp.Compile(headerName) if err != nil { - continue + return err, false } - headerNameMatch := regex.FindStringSubmatch(headerNameValue) + headerNameMatch := regex.MatchString(bannedHeaderName) + regex, err = regexp.Compile(bannedHeaderValue) if err != nil { - continue + return err, false } - headerValueMatch := regex.FindStringSubmatch(headerNameValue) - if len(headerNameMatch) > 0 || len(headerValueMatch) > 0 || headerName == bannedHeaderName || bannedHeaderValue == headerNameValue { - return true + headerValueMatch := regex.MatchString(headerNameValue) + if headerNameMatch || headerValueMatch { + return nil, true } } } - return false + return nil, false } func (ys *Yaml) WriteTestcase(tcRead platform.KindSpecifier, ctx context.Context, filtersRead platform.KindSpecifier) error { @@ -193,10 +194,14 @@ func (ys *Yaml) WriteTestcase(tcRead platform.KindSpecifier, ctx context.Context if ok { for _, testFilter := range testFilters.Filters { - if containsMatchingUrl(testFilter.UrlMethods, testFilter.Path, tc.HttpReq.URL, tc.HttpReq.Method) { + if err, containsMatch := containsMatchingUrl(testFilter.UrlMethods, testFilter.Path, tc.HttpReq.URL, tc.HttpReq.Method); err == nil && containsMatch { bypassTestCase = true - } else if hasBannedHeaders(tc.HttpReq.Header, testFilter.Headers) { + } else if err != nil { + return fmt.Errorf("%s failed to check matching url, error %s", Emoji, err.Error()) + } else if bannerHeaderCheck, hasHeader := hasBannedHeaders(tc.HttpReq.Header, testFilter.Headers); bannerHeaderCheck == nil && hasHeader { bypassTestCase = true + } else if bannerHeaderCheck != nil { + return fmt.Errorf("%s failed to check banned header, error %s", Emoji, err.Error()) } } } diff --git a/pkg/proxy/integrations/grpcparser/grpc_parser.go b/pkg/proxy/integrations/grpcparser/grpc_parser.go index ea10208dd..c55b72eb9 100644 --- a/pkg/proxy/integrations/grpcparser/grpc_parser.go +++ b/pkg/proxy/integrations/grpcparser/grpc_parser.go @@ -36,7 +36,7 @@ func (g *GrpcParser) OutgoingType(buffer []byte) bool { return bytes.HasPrefix(buffer[:], []byte("PRI * HTTP/2")) } -func (g *GrpcParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { +func (g *GrpcParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { switch models.GetMode() { case models.MODE_RECORD: encodeOutgoingGRPC(requestBuffer, clientConn, destConn, g.hooks, g.logger, ctx) diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index 6b638e613..d29b2d19f 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -32,17 +32,17 @@ type HttpParser struct { } // ProcessOutgoing implements proxy.DepInterface. -func (http *HttpParser) ProcessOutgoing(request []byte, clientConn, destConn net.Conn, ctx context.Context) { +func (http *HttpParser) ProcessOutgoing(request []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { switch models.GetMode() { case models.MODE_RECORD: - err := encodeOutgoingHttp(request, clientConn, destConn, http.logger, http.hooks, ctx) + err := encodeOutgoingHttp(request, clientConn, destConn, http.logger, http.hooks, ctx, sourcePort) if err != nil { http.logger.Error("failed to encode the http message into the yaml", zap.Error(err)) return } case models.MODE_TEST: - decodeOutgoingHttp(request, clientConn, destConn, http.hooks, http.logger) + decodeOutgoingHttp(request, clientConn, destConn, http.hooks, http.logger, sourcePort) default: http.logger.Info("Invalid mode detected while intercepting outgoing http call", zap.Any("mode", models.GetMode())) } @@ -97,14 +97,14 @@ func mapsHaveSameKeys(map1 map[string]string, map2 map[string][]string) bool { func ProcessOutgoingHttp(request []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger, ctx context.Context) { switch models.GetMode() { case models.MODE_RECORD: - err := encodeOutgoingHttp(request, clientConn, destConn, logger, h, ctx) + err := encodeOutgoingHttp(request, clientConn, destConn, logger, h, ctx, 0) if err != nil { logger.Error("failed to encode the http message into the yaml", zap.Error(err)) return } case models.MODE_TEST: - decodeOutgoingHttp(request, clientConn, destConn, h, logger) + decodeOutgoingHttp(request, clientConn, destConn, h, logger, 0) default: logger.Info("Invalid mode detected while intercepting outgoing http call", zap.Any("mode", models.GetMode())) } @@ -431,7 +431,7 @@ func checkIfGzipped(check io.ReadCloser) (bool, *bufio.Reader) { } // Decodes the mocks in test mode so that they can be sent to the user application. -func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger) { +func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger, sourcePort int) { //Matching algorithmm //Get the mocks for { @@ -508,8 +508,8 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * if err != nil { continue } - subMatches := regex.FindStringSubmatch(req.URL.String()) - if len(subMatches) > 0 && host != "" || req.Host == host { + matches := regex.MatchString(req.URL.String()) + if matches && host != "" || req.Host == host { passthroughHost = true } @@ -519,8 +519,8 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * if err != nil { continue } - subMatches := regex.FindStringSubmatch(req.URL.String()) - if h.GetSourcePort() == int(filter.Port) && (len(subMatches) > 0 && filter.Path != "") { + matches := regex.MatchString(req.URL.String()) + if sourcePort == int(filter.Port) && (matches && filter.Path != "") { passthroughHost = true } } @@ -596,7 +596,7 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * } // encodeOutgoingHttp function parses the HTTP request and response text messages to capture outgoing network calls as mocks. -func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *zap.Logger, h *hooks.Hook, ctx context.Context) error { +func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *zap.Logger, h *hooks.Hook, ctx context.Context, sourcePort int) error { var resp []byte var finalResp []byte var finalReq []byte @@ -690,7 +690,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z } // saving last request/response on this connection. - err := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h) + err := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h, sourcePort) if err != nil { logger.Error("failed to parse the final http request and response", zap.Error(err)) return err @@ -721,7 +721,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z if err == io.EOF { logger.Debug("connection closed by the server", zap.Error(err)) //check if before EOF complete response came, and try to parse it. - parseErr := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h) + parseErr := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h, sourcePort) if parseErr != nil { logger.Error("failed to parse the final http request and response", zap.Error(parseErr)) return parseErr @@ -735,7 +735,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z logger.Debug("This is the final response: " + string(finalResp)) - err = ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h) + err = ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h, sourcePort) if err != nil { logger.Error("failed to parse the final http request and response", zap.Error(err)) return err @@ -764,7 +764,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z } // ParseFinalHttp is used to parse the final http request and response and save it in a yaml file -func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTimestampcMock time.Time, ctx context.Context, logger *zap.Logger, h *hooks.Hook) error { +func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTimestampcMock time.Time, ctx context.Context, logger *zap.Logger, h *hooks.Hook, sourcePort int) error { var req *http.Request // converts the request message buffer to http request req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(finalReq))) @@ -837,8 +837,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime if err != nil { continue } - subMatches := regex.FindStringSubmatch(req.URL.String()) - if len(subMatches) > 0 && host != "" || req.Host == host { + matches := regex.MatchString(req.URL.String()) + if matches && host != "" || req.Host == host { passthroughHost = true } } @@ -847,8 +847,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime if err != nil { continue } - subMatches := regex.FindStringSubmatch(req.URL.String()) - if h.GetSourcePort() == int(filter.Port) && (len(subMatches) > 0 && filter.Path != "") { + matches := regex.MatchString(req.URL.String()) + if sourcePort == int(filter.Port) && (matches && filter.Path != "") { passthroughHost = true } } diff --git a/pkg/proxy/integrations/mongoparser/mongoparser.go b/pkg/proxy/integrations/mongoparser/mongoparser.go index 39a5a0246..c7016766e 100644 --- a/pkg/proxy/integrations/mongoparser/mongoparser.go +++ b/pkg/proxy/integrations/mongoparser/mongoparser.go @@ -49,7 +49,7 @@ func (m *MongoParser) OutgoingType(buffer []byte) bool { return int(messageLength) == len(buffer) } -func (m *MongoParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { +func (m *MongoParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { switch models.GetMode() { case models.MODE_RECORD: m.logger.Debug("the outgoing mongo in record mode") diff --git a/pkg/proxy/integrations/mysqlparser/mysqlparser.go b/pkg/proxy/integrations/mysqlparser/mysqlparser.go index 883e49879..a58e1d9b7 100644 --- a/pkg/proxy/integrations/mysqlparser/mysqlparser.go +++ b/pkg/proxy/integrations/mysqlparser/mysqlparser.go @@ -32,7 +32,7 @@ func (sql *MySqlParser) OutgoingType(buffer []byte) bool { //Returning false here because sql parser is using the ports to check if the packet is mysql or not. return false } -func (sql *MySqlParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { +func (sql *MySqlParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { delay := sql.delay switch models.GetMode() { case models.MODE_RECORD: diff --git a/pkg/proxy/integrations/postgresParser/postgres_parser.go b/pkg/proxy/integrations/postgresParser/postgres_parser.go index adb4f2fa0..81654f53e 100755 --- a/pkg/proxy/integrations/postgresParser/postgres_parser.go +++ b/pkg/proxy/integrations/postgresParser/postgres_parser.go @@ -58,7 +58,7 @@ func (p *PostgresParser) OutgoingType(buffer []byte) bool { return version == ProtocolVersion } -func (p *PostgresParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { +func (p *PostgresParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { switch models.GetMode() { case models.MODE_RECORD: encodePostgresOutgoing(requestBuffer, clientConn, destConn, p.hooks, p.logger, ctx) diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index c2793c988..46ed73809 100755 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -49,7 +49,7 @@ var Emoji = "\U0001F430" + " Keploy:" type DependencyHandler interface { OutgoingType(buffer []byte) bool - ProcessOutgoing(buffer []byte, conn net.Conn, dst net.Conn, ctx context.Context) + ProcessOutgoing(buffer []byte, conn net.Conn, dst net.Conn, ctx context.Context, sourcePort int) } var ParsersMap = make(map[string]DependencyHandler) @@ -775,8 +775,6 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con remoteAddr := conn.RemoteAddr().(*net.TCPAddr) sourcePort := remoteAddr.Port - ps.hook.SetSourcePort(sourcePort) - ps.logger.Debug("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) //TODO: fix this bug, getting source port same as proxy port. @@ -817,7 +815,7 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con // } } } - ParsersMap["mysql"].ProcessOutgoing([]byte{}, conn, dst, ctx) + ParsersMap["mysql"].ProcessOutgoing([]byte{}, conn, dst, ctx, sourcePort) } else { clientConnId := util.GetNextID() @@ -914,7 +912,7 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con //Checking for all the parsers. for _, parser := range ParsersMap { if parser.OutgoingType(buffer) { - parser.ProcessOutgoing(buffer, conn, dst, ctx) + parser.ProcessOutgoing(buffer, conn, dst, ctx, sourcePort) genericCheck = false } } diff --git a/pkg/service/generateConfig/generateConfig.go b/pkg/service/generateConfig/generateConfig.go index a85dce387..29cb4f809 100644 --- a/pkg/service/generateConfig/generateConfig.go +++ b/pkg/service/generateConfig/generateConfig.go @@ -76,51 +76,6 @@ test: port: 0 withCoverage: false coverageReportPath: "" - # Example on using tests - # tests: - # filters: - # - path: "/user/app" - # urlMethods: ["GET"] - # headers: { - # "^asdf*": "^test" - # } - # host: "dc.services.visualstudio.com" - # Example on using stubs - # stubs: - # filters: - # - path: "/user/app" - # port: 8080 - # - port: 8081 - # - host: "dc.services.visualstudio.com" - # - port: 8081 - # host: "dc.services.visualstudio.com" - # path: "/user/app" - # - # Example on using globalNoise - # globalNoise: - # global: - # body: { - # # to ignore some values for a field, - # # pass regex patterns to the corresponding array value - # "url": ["https?://\S+", "http://\S+"], - # } - # header: { - # # to ignore the entire field, pass an empty array - # "Date": [], - # } - # # to ignore fields or the corresponding values for a specific test-set, - # # pass the test-set-name as a key to the "test-sets" object and - # # populate the corresponding "body" and "header" objects - # test-sets: - # test-set-1: - # body: { - # # ignore all the values for the "url" field - # "url": [] - # } - # header: { - # # we can also pass the exact value to ignore for a field - # "User-Agent": ["PostmanRuntime/7.34.0"] - # } ` func (g *generatorConfig) GenerateConfig(filePath string) { diff --git a/utils/utils.go b/utils/utils.go index d3eda6a7a..4f82d7968 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -16,31 +16,51 @@ import ( var Emoji = "\U0001F430" + " Keploy:" var ConfigGuide = ` - # Example on using globalNoise - # globalNoise: - # global: - # body: { - # # to ignore some values for a field, - # # pass regex patterns to the corresponding array value - # "url": ["https?://\S+", "http://\S+"], - # } - # header: { - # # to ignore the entire field, pass an empty array - # "Date": [], - # } - # # to ignore fields or the corresponding values for a specific test-set, - # # pass the test-set-name as a key to the "test-sets" object and - # # populate the corresponding "body" and "header" objects - # test-sets: - # test-set-1: - # body: { - # # ignore all the values for the "url" field - # "url": [] - # } - # header: { - # # we can also pass the exact value to ignore for a field - # "User-Agent": ["PostmanRuntime/7.34.0"] - # } + # Example on using tests + # tests: + # filters: + # - path: "/user/app" + # urlMethods: ["GET"] + # headers: { + # "^asdf*": "^test" + # } + # host: "dc.services.visualstudio.com" + # Example on using stubs + # stubs: + # filters: + # - path: "/user/app" + # port: 8080 + # - port: 8081 + # - host: "dc.services.visualstudio.com" + # - port: 8081 + # host: "dc.services.visualstudio.com" + # path: "/user/app" + # + # Example on using globalNoise + # globalNoise: + # global: + # body: { + # # to ignore some values for a field, + # # pass regex patterns to the corresponding array value + # "url": ["https?://\S+", "http://\S+"], + # } + # header: { + # # to ignore the entire field, pass an empty array + # "Date": [], + # } + # # to ignore fields or the corresponding values for a specific test-set, + # # pass the test-set-name as a key to the "test-sets" object and + # # populate the corresponding "body" and "header" objects + # test-sets: + # test-set-1: + # body: { + # # ignore all the values for the "url" field + # "url": [] + # } + # header: { + # # we can also pass the exact value to ignore for a field + # "User-Agent": ["PostmanRuntime/7.34.0"] + # } ` // askForConfirmation asks the user for confirmation. A user must type in "yes" or "no" and From 4f175c4ef11619cdfeeab44d918350ddd27c7684 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 22 Jan 2024 10:14:22 +0000 Subject: [PATCH 09/20] fix:yam --- utils/utils.go | 88 +++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index 4f82d7968..3ea289213 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -16,51 +16,51 @@ import ( var Emoji = "\U0001F430" + " Keploy:" var ConfigGuide = ` - # Example on using tests - # tests: - # filters: - # - path: "/user/app" - # urlMethods: ["GET"] - # headers: { - # "^asdf*": "^test" - # } - # host: "dc.services.visualstudio.com" - # Example on using stubs - # stubs: - # filters: - # - path: "/user/app" - # port: 8080 - # - port: 8081 - # - host: "dc.services.visualstudio.com" - # - port: 8081 - # host: "dc.services.visualstudio.com" - # path: "/user/app" +# Example on using tests +#tests: +# filters: +# - path: "/user/app" +# urlMethods: ["GET"] +# headers: { +# "^asdf*": "^test" +# } +# host: "dc.services.visualstudio.com" +#Example on using stubs +#stubs: +# filters: +# - path: "/user/app" +# port: 8080 +# - port: 8081 +# - host: "dc.services.visualstudio.com" +# - port: 8081 +# host: "dc.services.visualstudio.com" +# path: "/user/app" # - # Example on using globalNoise - # globalNoise: - # global: - # body: { - # # to ignore some values for a field, - # # pass regex patterns to the corresponding array value - # "url": ["https?://\S+", "http://\S+"], - # } - # header: { - # # to ignore the entire field, pass an empty array - # "Date": [], - # } - # # to ignore fields or the corresponding values for a specific test-set, - # # pass the test-set-name as a key to the "test-sets" object and - # # populate the corresponding "body" and "header" objects - # test-sets: - # test-set-1: - # body: { - # # ignore all the values for the "url" field - # "url": [] - # } - # header: { - # # we can also pass the exact value to ignore for a field - # "User-Agent": ["PostmanRuntime/7.34.0"] - # } +#Example on using globalNoise +#globalNoise: +# global: +# body: { +# # to ignore some values for a field, +# # pass regex patterns to the corresponding array value +# "url": ["https?://\S+", "http://\S+"], +# } +# header: { +# # to ignore the entire field, pass an empty array +# "Date": [], +# } +# # to ignore fields or the corresponding values for a specific test-set, +# # pass the test-set-name as a key to the "test-sets" object and +# # populate the corresponding "body" and "header" objects +# test-sets: +# test-set-1: +# body: { +# # ignore all the values for the "url" field +# "url": [] +# } +# header: { +# # we can also pass the exact value to ignore for a field +# "User-Agent": ["PostmanRuntime/7.34.0"] +# } ` // askForConfirmation asks the user for confirmation. A user must type in "yes" or "no" and From cf5524ede9152f6792804a8c7f1e334446b089c6 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 22 Jan 2024 10:22:02 +0000 Subject: [PATCH 10/20] udpate:append --- cmd/test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/test.go b/cmd/test.go index b126e0fad..37ae7b97d 100755 --- a/cmd/test.go +++ b/cmd/test.go @@ -90,10 +90,10 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te if !*ignoreOrdering { *ignoreOrdering = confTest.IgnoreOrdering } - *passThroughHosts = append(*passThroughHosts, confTest.Stubs.Filters...) + passThroughPortProvided := len(*passThroughPorts) == 0 if len(*passThroughPorts) == 0 { for _, filter := range confTest.Stubs.Filters { - if filter.Port != 0 && filter.Host == "" && filter.Path == "" { + if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided { *passThroughPorts = append(*passThroughPorts, filter.Port) } else { *passThroughHosts = append(*passThroughHosts, filter) From 4e292ebd0af312c32de4037e5342e34b1c7beeab Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 29 Jan 2024 07:43:40 +0000 Subject: [PATCH 11/20] refactor:log --- cmd/record.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/record.go b/cmd/record.go index 586b6e2f4..4fcdee931 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -170,7 +170,7 @@ func (r *Record) GetCmd() *cobra.Command { err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, &passThrough, configPath) if err != nil { if err == errFileNotFound { - r.logger.Info("Keploy config not found, using default config") + r.logger.Info("Keploy config not found, ontinuing without configuration") } else { r.logger.Error("", zap.Error(err)) } From c217d710074f43bc1d05e5b2a2fff7908afc1700 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 29 Jan 2024 07:51:46 +0000 Subject: [PATCH 12/20] refactor:url match --- pkg/platform/yaml/yaml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/platform/yaml/yaml.go b/pkg/platform/yaml/yaml.go index 2f507c399..563a00a4c 100755 --- a/pkg/platform/yaml/yaml.go +++ b/pkg/platform/yaml/yaml.go @@ -149,7 +149,7 @@ func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, urlMatched = true } - if len(urlMethods) != 0 { + if len(urlMethods) != 0 && urlMatched { urlMatched = false for _, method := range urlMethods { if string(method) == string(requestMethod) { From ed074222b083ff8c121b68d589fd1dcb3dd7679a Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 29 Jan 2024 08:07:22 +0000 Subject: [PATCH 13/20] refactor:logic to update passthrough hosts --- cmd/test.go | 12 +++++------- pkg/graph/serve.go | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/test.go b/cmd/test.go index 37ae7b97d..40f86a652 100755 --- a/cmd/test.go +++ b/cmd/test.go @@ -91,13 +91,11 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te *ignoreOrdering = confTest.IgnoreOrdering } passThroughPortProvided := len(*passThroughPorts) == 0 - if len(*passThroughPorts) == 0 { - for _, filter := range confTest.Stubs.Filters { - if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided { - *passThroughPorts = append(*passThroughPorts, filter.Port) - } else { - *passThroughHosts = append(*passThroughHosts, filter) - } + for _, filter := range confTest.Stubs.Filters { + if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided { + *passThroughPorts = append(*passThroughPorts, filter.Port) + } else { + *passThroughHosts = append(*passThroughHosts, filter) } } diff --git a/pkg/graph/serve.go b/pkg/graph/serve.go index 269025364..4245cb2f1 100644 --- a/pkg/graph/serve.go +++ b/pkg/graph/serve.go @@ -202,7 +202,7 @@ func (g *graph) Serve(path string, proxyPort uint32, testReportPath string, Dela } // Gracefully shut down the HTTP server with a timeout -func (g *graph)stopGraphqlServer(httpSrv *http.Server) { +func (g *graph) stopGraphqlServer(httpSrv *http.Server) { shutdown := make(chan struct{}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -211,4 +211,4 @@ func (g *graph)stopGraphqlServer(httpSrv *http.Server) { } // If you have other goroutines that should listen for this, you can use this channel to notify them. close(shutdown) -} \ No newline at end of file +} From d08488576cb5c7d1324d73c7056b51c41eb7f13d Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 29 Jan 2024 10:31:37 +0000 Subject: [PATCH 14/20] refactor:update function name,move source port logic --- pkg/hooks/loader.go | 4 ++-- .../integrations/grpcparser/grpc_parser.go | 2 +- .../integrations/httpparser/httpparser.go | 23 +++++++++++-------- .../integrations/mongoparser/mongoparser.go | 2 +- .../integrations/mysqlparser/mysqlparser.go | 2 +- .../postgresParser/postgres_parser.go | 2 +- pkg/proxy/proxy.go | 6 ++--- pkg/service/record/record.go | 2 +- pkg/service/test/test.go | 2 +- 9 files changed, 24 insertions(+), 21 deletions(-) diff --git a/pkg/hooks/loader.go b/pkg/hooks/loader.go index 19d7727c0..58dde424b 100755 --- a/pkg/hooks/loader.go +++ b/pkg/hooks/loader.go @@ -142,11 +142,11 @@ func (h *Hook) GetProxyPort() uint32 { return h.proxyPort } -func (h *Hook) GetProxyHost() models.Stubs { +func (h *Hook) GetPassThroughHosts() models.Stubs { return h.passThroughHosts } -func (h *Hook) SetProxyHosts(passThroughHosts []models.Filters) { +func (h *Hook) SetPassThroughHosts(passThroughHosts []models.Filters) { h.passThroughHosts = models.Stubs{ Filters: passThroughHosts, } diff --git a/pkg/proxy/integrations/grpcparser/grpc_parser.go b/pkg/proxy/integrations/grpcparser/grpc_parser.go index c55b72eb9..ea10208dd 100644 --- a/pkg/proxy/integrations/grpcparser/grpc_parser.go +++ b/pkg/proxy/integrations/grpcparser/grpc_parser.go @@ -36,7 +36,7 @@ func (g *GrpcParser) OutgoingType(buffer []byte) bool { return bytes.HasPrefix(buffer[:], []byte("PRI * HTTP/2")) } -func (g *GrpcParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { +func (g *GrpcParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { switch models.GetMode() { case models.MODE_RECORD: encodeOutgoingGRPC(requestBuffer, clientConn, destConn, g.hooks, g.logger, ctx) diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index d29b2d19f..28ab0ec2f 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -32,17 +32,17 @@ type HttpParser struct { } // ProcessOutgoing implements proxy.DepInterface. -func (http *HttpParser) ProcessOutgoing(request []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { +func (http *HttpParser) ProcessOutgoing(request []byte, clientConn, destConn net.Conn, ctx context.Context) { switch models.GetMode() { case models.MODE_RECORD: - err := encodeOutgoingHttp(request, clientConn, destConn, http.logger, http.hooks, ctx, sourcePort) + err := encodeOutgoingHttp(request, clientConn, destConn, http.logger, http.hooks, ctx) if err != nil { http.logger.Error("failed to encode the http message into the yaml", zap.Error(err)) return } case models.MODE_TEST: - decodeOutgoingHttp(request, clientConn, destConn, http.hooks, http.logger, sourcePort) + decodeOutgoingHttp(request, clientConn, destConn, http.hooks, http.logger) default: http.logger.Info("Invalid mode detected while intercepting outgoing http call", zap.Any("mode", models.GetMode())) } @@ -97,14 +97,14 @@ func mapsHaveSameKeys(map1 map[string]string, map2 map[string][]string) bool { func ProcessOutgoingHttp(request []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger, ctx context.Context) { switch models.GetMode() { case models.MODE_RECORD: - err := encodeOutgoingHttp(request, clientConn, destConn, logger, h, ctx, 0) + err := encodeOutgoingHttp(request, clientConn, destConn, logger, h, ctx) if err != nil { logger.Error("failed to encode the http message into the yaml", zap.Error(err)) return } case models.MODE_TEST: - decodeOutgoingHttp(request, clientConn, destConn, h, logger, 0) + decodeOutgoingHttp(request, clientConn, destConn, h, logger) default: logger.Info("Invalid mode detected while intercepting outgoing http call", zap.Any("mode", models.GetMode())) } @@ -431,11 +431,12 @@ func checkIfGzipped(check io.ReadCloser) (bool, *bufio.Reader) { } // Decodes the mocks in test mode so that they can be sent to the user application. -func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger, sourcePort int) { +func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger) { //Matching algorithmm //Get the mocks for { - + remoteAddr := clientConn.RemoteAddr().(*net.TCPAddr) + sourcePort := remoteAddr.Port //Check if the expected header is present if bytes.Contains(requestBuffer, []byte("Expect: 100-continue")) { //Send the 100 continue response @@ -494,7 +495,7 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * passthroughHost := false passThrough := models.PassThroughHosts portPassThrough := []models.Filters{} - for _, filters := range h.GetProxyHost().Filters { + for _, filters := range h.GetPassThroughHosts().Filters { if filters.Port == 0 { passThrough = append(passThrough, filters.Host, filters.Path) } else { @@ -596,12 +597,14 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * } // encodeOutgoingHttp function parses the HTTP request and response text messages to capture outgoing network calls as mocks. -func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *zap.Logger, h *hooks.Hook, ctx context.Context, sourcePort int) error { +func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *zap.Logger, h *hooks.Hook, ctx context.Context) error { var resp []byte var finalResp []byte var finalReq []byte var err error + remoteAddr := clientConn.RemoteAddr().(*net.TCPAddr) + sourcePort := remoteAddr.Port //closing the destination connection defer destConn.Close() @@ -825,7 +828,7 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime passthroughHost := false passThrough := models.PassThroughHosts portPassThrough := []models.Filters{} - for _, filters := range h.GetProxyHost().Filters { + for _, filters := range h.GetPassThroughHosts().Filters { if filters.Port == 0 { passThrough = append(passThrough, filters.Host, filters.Path) } else { diff --git a/pkg/proxy/integrations/mongoparser/mongoparser.go b/pkg/proxy/integrations/mongoparser/mongoparser.go index c7016766e..39a5a0246 100644 --- a/pkg/proxy/integrations/mongoparser/mongoparser.go +++ b/pkg/proxy/integrations/mongoparser/mongoparser.go @@ -49,7 +49,7 @@ func (m *MongoParser) OutgoingType(buffer []byte) bool { return int(messageLength) == len(buffer) } -func (m *MongoParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { +func (m *MongoParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { switch models.GetMode() { case models.MODE_RECORD: m.logger.Debug("the outgoing mongo in record mode") diff --git a/pkg/proxy/integrations/mysqlparser/mysqlparser.go b/pkg/proxy/integrations/mysqlparser/mysqlparser.go index a58e1d9b7..883e49879 100644 --- a/pkg/proxy/integrations/mysqlparser/mysqlparser.go +++ b/pkg/proxy/integrations/mysqlparser/mysqlparser.go @@ -32,7 +32,7 @@ func (sql *MySqlParser) OutgoingType(buffer []byte) bool { //Returning false here because sql parser is using the ports to check if the packet is mysql or not. return false } -func (sql *MySqlParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { +func (sql *MySqlParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { delay := sql.delay switch models.GetMode() { case models.MODE_RECORD: diff --git a/pkg/proxy/integrations/postgresParser/postgres_parser.go b/pkg/proxy/integrations/postgresParser/postgres_parser.go index 81654f53e..adb4f2fa0 100755 --- a/pkg/proxy/integrations/postgresParser/postgres_parser.go +++ b/pkg/proxy/integrations/postgresParser/postgres_parser.go @@ -58,7 +58,7 @@ func (p *PostgresParser) OutgoingType(buffer []byte) bool { return version == ProtocolVersion } -func (p *PostgresParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) { +func (p *PostgresParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) { switch models.GetMode() { case models.MODE_RECORD: encodePostgresOutgoing(requestBuffer, clientConn, destConn, p.hooks, p.logger, ctx) diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index 46ed73809..95e5bd7b7 100755 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -49,7 +49,7 @@ var Emoji = "\U0001F430" + " Keploy:" type DependencyHandler interface { OutgoingType(buffer []byte) bool - ProcessOutgoing(buffer []byte, conn net.Conn, dst net.Conn, ctx context.Context, sourcePort int) + ProcessOutgoing(buffer []byte, conn net.Conn, dst net.Conn, ctx context.Context) } var ParsersMap = make(map[string]DependencyHandler) @@ -815,7 +815,7 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con // } } } - ParsersMap["mysql"].ProcessOutgoing([]byte{}, conn, dst, ctx, sourcePort) + ParsersMap["mysql"].ProcessOutgoing([]byte{}, conn, dst, ctx) } else { clientConnId := util.GetNextID() @@ -912,7 +912,7 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con //Checking for all the parsers. for _, parser := range ParsersMap { if parser.OutgoingType(buffer) { - parser.ProcessOutgoing(buffer, conn, dst, ctx, sourcePort) + parser.ProcessOutgoing(buffer, conn, dst, ctx) genericCheck = false } } diff --git a/pkg/service/record/record.go b/pkg/service/record/record.go index a71ae73b8..8a829baa4 100755 --- a/pkg/service/record/record.go +++ b/pkg/service/record/record.go @@ -50,7 +50,7 @@ func (r *recorder) CaptureTraffic(path string, proxyPort uint32, appCmd, appCont routineId := pkg.GenerateRandomID() // Initiate the hooks and update the vaccant ProxyPorts map loadedHooks, err := hooks.NewHook(ys, routineId, r.Logger) - loadedHooks.SetProxyHosts(passThroughHosts) + loadedHooks.SetPassThroughHosts(passThroughHosts) if err != nil { r.Logger.Error("error while creating hooks", zap.Error(err)) return diff --git a/pkg/service/test/test.go b/pkg/service/test/test.go index ed65ee418..606580ba7 100755 --- a/pkg/service/test/test.go +++ b/pkg/service/test/test.go @@ -125,7 +125,7 @@ func (t *tester) InitialiseTest(cfg *TestConfig) (InitialiseTestReturn, error) { routineId := pkg.GenerateRandomID() // Initiate the hooks returnVal.LoadedHooks, err = hooks.NewHook(returnVal.YamlStore, routineId, t.logger) - returnVal.LoadedHooks.SetProxyHosts(cfg.PassThroughHosts) + returnVal.LoadedHooks.SetPassThroughHosts(cfg.PassThroughHosts) if err != nil { return returnVal, fmt.Errorf("error while creating hooks %v", err) } From 0a44c8d6683d21d8ce2a0ed63c0b47f0042529cc Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Mon, 29 Jan 2024 11:46:36 +0000 Subject: [PATCH 15/20] refactor:removing filters from tests --- pkg/service/generateConfig/generateConfig.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/service/generateConfig/generateConfig.go b/pkg/service/generateConfig/generateConfig.go index 29cb4f809..21285b60e 100644 --- a/pkg/service/generateConfig/generateConfig.go +++ b/pkg/service/generateConfig/generateConfig.go @@ -63,12 +63,6 @@ test: buildDelay: 30s apiTimeout: 5 ignoreOrdering: false - tests: - filters: - - path: "" - urlMethods: [] - headers: {} - host: "" stubs: filters: - path: "" From 9c95727c4ac0ae20a656cd2f36770df569eef195 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Tue, 30 Jan 2024 02:47:13 +0000 Subject: [PATCH 16/20] refactor:logs and config --- cmd/record.go | 2 +- cmd/test.go | 2 +- pkg/platform/yaml/yaml.go | 2 +- pkg/proxy/integrations/httpparser/httpparser.go | 9 ++++++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/record.go b/cmd/record.go index 4fcdee931..9039a4a16 100755 --- a/cmd/record.go +++ b/cmd/record.go @@ -170,7 +170,7 @@ func (r *Record) GetCmd() *cobra.Command { err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, &passThrough, configPath) if err != nil { if err == errFileNotFound { - r.logger.Info("Keploy config not found, ontinuing without configuration") + r.logger.Info("Keploy config not found, continuing without configuration") } else { r.logger.Error("", zap.Error(err)) } diff --git a/cmd/test.go b/cmd/test.go index 40f86a652..7724575ee 100755 --- a/cmd/test.go +++ b/cmd/test.go @@ -253,7 +253,7 @@ func (t *Test) GetCmd() *cobra.Command { err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering, &passThroughHosts) if err != nil { if err == errFileNotFound { - t.logger.Info("Keploy config not found, using default config") + t.logger.Info("Keploy config not found, continuing without configuration") } else { t.logger.Error("", zap.Error(err)) } diff --git a/pkg/platform/yaml/yaml.go b/pkg/platform/yaml/yaml.go index 563a00a4c..2b262c8fd 100755 --- a/pkg/platform/yaml/yaml.go +++ b/pkg/platform/yaml/yaml.go @@ -175,7 +175,7 @@ func hasBannedHeaders(object map[string]string, bannedHeaders map[string]string) return err, false } headerValueMatch := regex.MatchString(headerNameValue) - if headerNameMatch || headerValueMatch { + if headerNameMatch && headerValueMatch { return nil, true } } diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index 28ab0ec2f..af25f0183 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -502,27 +502,30 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * portPassThrough = append(portPassThrough, filters) } } - // Define the regular expression pattern for _, host := range passThrough { regex, err := regexp.Compile(host) if err != nil { + logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) continue } matches := regex.MatchString(req.URL.String()) if matches && host != "" || req.Host == host { passthroughHost = true + break } } for _, filter := range portPassThrough { regex, err := regexp.Compile(filter.Path) if err != nil { + logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) continue } matches := regex.MatchString(req.URL.String()) if sourcePort == int(filter.Port) && (matches && filter.Path != "") { passthroughHost = true + break } } if !passthroughHost { @@ -838,21 +841,25 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime for _, host := range passThrough { regex, err := regexp.Compile(host) if err != nil { + logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) continue } matches := regex.MatchString(req.URL.String()) if matches && host != "" || req.Host == host { passthroughHost = true + break } } for _, filter := range portPassThrough { regex, err := regexp.Compile(filter.Path) if err != nil { + logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) continue } matches := regex.MatchString(req.URL.String()) if sourcePort == int(filter.Port) && (matches && filter.Path != "") { passthroughHost = true + break } } if !passthroughHost { From 0c7bff4652cdee102cb1428d318c56094ca3e8df Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Tue, 30 Jan 2024 02:57:06 +0000 Subject: [PATCH 17/20] fix:merge errors --- pkg/models/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/models/config.go b/pkg/models/config.go index 45ec57fc3..f5db0e8bd 100644 --- a/pkg/models/config.go +++ b/pkg/models/config.go @@ -44,7 +44,7 @@ type Test struct { ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` ContainerName string `json:"containerName" yaml:"containerName"` NetworkName string `json:"networkName" yaml:"networkName"` - Tests map[string][]string `json:"tests" yaml:"tests"` + SelectedTests map[string][]string `json:"tests" yaml:"tests"` GlobalNoise Globalnoise `json:"globalNoise" yaml:"globalNoise"` Delay uint64 `json:"delay" yaml:"delay"` BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` @@ -54,6 +54,7 @@ type Test struct { WithCoverage bool `json:"withCoverage" yaml:"withCoverage"` // boolean to capture the coverage in test CoverageReportPath string `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files IgnoreOrdering bool `json:"ignoreOrdering" yaml:"ignoreOrdering"` + Stubs Stubs `json:"stubs" yaml:"stubs"` } type Globalnoise struct { From 478abe53c2b6b0ee75dc1a78a047f1117646e0d8 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Tue, 30 Jan 2024 03:08:46 +0000 Subject: [PATCH 18/20] fix:selected tests in config --- pkg/models/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/config.go b/pkg/models/config.go index f5db0e8bd..6b84556da 100644 --- a/pkg/models/config.go +++ b/pkg/models/config.go @@ -44,7 +44,7 @@ type Test struct { ProxyPort uint32 `json:"proxyport" yaml:"proxyport"` ContainerName string `json:"containerName" yaml:"containerName"` NetworkName string `json:"networkName" yaml:"networkName"` - SelectedTests map[string][]string `json:"tests" yaml:"tests"` + SelectedTests map[string][]string `json:"selectedTests" yaml:"selectedTests"` GlobalNoise Globalnoise `json:"globalNoise" yaml:"globalNoise"` Delay uint64 `json:"delay" yaml:"delay"` BuildDelay time.Duration `json:"buildDelay" yaml:"buildDelay"` From 3bbadcb59f13e210af52547946875b17b80cb12c Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Tue, 30 Jan 2024 05:24:02 +0000 Subject: [PATCH 19/20] refactor:passthrough host and path --- .../integrations/httpparser/httpparser.go | 104 +++++++++--------- 1 file changed, 49 insertions(+), 55 deletions(-) diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index af25f0183..5cdef04c0 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -493,39 +493,35 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h * } if !isMatched { passthroughHost := false - passThrough := models.PassThroughHosts - portPassThrough := []models.Filters{} for _, filters := range h.GetPassThroughHosts().Filters { - if filters.Port == 0 { - passThrough = append(passThrough, filters.Host, filters.Path) - } else { - portPassThrough = append(portPassThrough, filters) - } - } - - for _, host := range passThrough { - regex, err := regexp.Compile(host) - if err != nil { - logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) - continue + if filters.Host != "" { + regex, err := regexp.Compile(filters.Host) + if err != nil { + logger.Error("failed to compile the host regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) + continue + } + passthroughHost = regex.MatchString(req.Host) + if !passthroughHost { + continue + } } - matches := regex.MatchString(req.URL.String()) - if matches && host != "" || req.Host == host { - passthroughHost = true - break + if filters.Path != "" { + regex, err := regexp.Compile(filters.Path) + if err != nil { + logger.Error("failed to compile the path regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) + continue + } + passthroughHost = regex.MatchString(req.URL.String()) + if !passthroughHost { + continue + } } - } - for _, filter := range portPassThrough { - regex, err := regexp.Compile(filter.Path) - if err != nil { - logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) - continue - } - matches := regex.MatchString(req.URL.String()) - if sourcePort == int(filter.Port) && (matches && filter.Path != "") { - passthroughHost = true + portSatisfied := filters.Port != 0 && sourcePort == int(filters.Port) + if (portSatisfied && passthroughHost) || (filters.Port == 0 && passthroughHost) { break + } else if filters.Port != 0 { + passthroughHost = false } } if !passthroughHost { @@ -829,37 +825,35 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime "operation": req.Method, } passthroughHost := false - passThrough := models.PassThroughHosts - portPassThrough := []models.Filters{} for _, filters := range h.GetPassThroughHosts().Filters { - if filters.Port == 0 { - passThrough = append(passThrough, filters.Host, filters.Path) - } else { - portPassThrough = append(portPassThrough, filters) - } - } - for _, host := range passThrough { - regex, err := regexp.Compile(host) - if err != nil { - logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) - continue - } - matches := regex.MatchString(req.URL.String()) - if matches && host != "" || req.Host == host { - passthroughHost = true - break + if filters.Host != "" { + regex, err := regexp.Compile(filters.Host) + if err != nil { + logger.Error("failed to compile the host regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) + continue + } + passthroughHost = regex.MatchString(req.Host) + if !passthroughHost { + continue + } } - } - for _, filter := range portPassThrough { - regex, err := regexp.Compile(filter.Path) - if err != nil { - logger.Error("failed to compile the regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) - continue + if filters.Path != "" { + regex, err := regexp.Compile(filters.Path) + if err != nil { + logger.Error("failed to compile the path regex", zap.Any("metadata", getReqMeta(req)), zap.Error(err)) + continue + } + passthroughHost = regex.MatchString(req.URL.String()) + if !passthroughHost { + continue + } } - matches := regex.MatchString(req.URL.String()) - if sourcePort == int(filter.Port) && (matches && filter.Path != "") { - passthroughHost = true + + portSatisfied := filters.Port != 0 && sourcePort == int(filters.Port) + if (portSatisfied && passthroughHost) || (filters.Port == 0 && passthroughHost) { break + } else if filters.Port != 0 { + passthroughHost = false } } if !passthroughHost { From 56fbb528c3270effe3446a3d38f66496b0fec1d7 Mon Sep 17 00:00:00 2001 From: shivamsouravjha Date: Tue, 30 Jan 2024 05:24:43 +0000 Subject: [PATCH 20/20] update:name of variable --- pkg/proxy/integrations/httpparser/httpparser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/proxy/integrations/httpparser/httpparser.go b/pkg/proxy/integrations/httpparser/httpparser.go index 5cdef04c0..5ceab214c 100755 --- a/pkg/proxy/integrations/httpparser/httpparser.go +++ b/pkg/proxy/integrations/httpparser/httpparser.go @@ -849,8 +849,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime } } - portSatisfied := filters.Port != 0 && sourcePort == int(filters.Port) - if (portSatisfied && passthroughHost) || (filters.Port == 0 && passthroughHost) { + portMatched := filters.Port != 0 && sourcePort == int(filters.Port) + if (portMatched && passthroughHost) || (filters.Port == 0 && passthroughHost) { break } else if filters.Port != 0 { passthroughHost = false