feat:support of pass throught hosts

.github/workflows/test_workflow_scripts/node-linux.sh

@@ -74,7 +74,7 @@ sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 --testset
 
 # Update the global noise to ts.
 config_file="./keploy-config.yaml"
-sed -i '/tests:/a \        "test-set-0": ["test-1", "test-2"]' "$config_file"
+sed -i '/selectedTests:/a \        "test-set-0": ["test-1", "test-2"]' "$config_file"
 
 
 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --apiTimeout 30 --delay 10

cmd/record.go

@@ -40,9 +40,9 @@ func readRecordConfig(configPath string) (*models.Record, error) {
 	return &doc.Record, nil
 }
 
-var filters = models.Filters{}
+var filters = models.TestFilter{}
 
-func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, configPath string) error {
+func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, passThrough *[]models.Filters, configPath string) error {
 	configFilePath := filepath.Join(configPath, "keploy-config.yaml")
 	if isExist := utils.CheckFileExists(configFilePath); !isExist {
 		return errFileNotFound
@@ -57,7 +57,9 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string
 	if len(*path) == 0 {
 		*path = confRecord.Path
 	}
-	filters = confRecord.Filters
+
+	filters = confRecord.Tests
+
 	if *proxyPort == 0 {
 		*proxyPort = confRecord.ProxyPort
 	}
@@ -76,9 +78,16 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string
 	if *buildDelay == 30*time.Second && confRecord.BuildDelay != 0 {
 		*buildDelay = confRecord.BuildDelay
 	}
-	if len(*passThroughPorts) == 0 {
-		*passThroughPorts = confRecord.PassThroughPorts
+	passThroughPortProvided := len(*passThroughPorts) == 0
+
+	for _, filter := range confRecord.Stubs.Filters {
+		if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided {
+			*passThroughPorts = append(*passThroughPorts, filter.Port)
+		} else {
+			*passThrough = append(*passThrough, filter)
+		}
 	}
+
 	return nil
 }
 
@@ -156,10 +165,12 @@ func (r *Record) GetCmd() *cobra.Command {
 				return err
 			}
 
-			err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, configPath)
+			passThrough := []models.Filters{}
+
+			err = r.GetRecordConfig(&path, &proxyPort, &appCmd, &appContainer, &networkName, &delay, &buildDelay, &ports, &passThrough, configPath)
 			if err != nil {
 				if err == errFileNotFound {
-					r.logger.Info("continuing without configuration file because file not found")
+					r.logger.Info("Keploy config not found, continuing without configuration")
 				} else {
 					r.logger.Error("", zap.Error(err))
 				}
@@ -257,7 +268,7 @@ func (r *Record) GetCmd() *cobra.Command {
 			}
 
 			r.logger.Debug("the ports are", zap.Any("ports", ports))
-			r.recorder.CaptureTraffic(path, proxyPort, appCmd, appContainer, networkName, delay, buildDelay, ports, &filters, enableTele)
+			r.recorder.CaptureTraffic(path, proxyPort, appCmd, appContainer, networkName, delay, buildDelay, ports, &filters, enableTele, passThrough)
 			return nil
 		},
 	}

cmd/test.go

@@ -42,7 +42,7 @@ func ReadTestConfig(configPath string) (*models.Test, error) {
 	return &doc.Test, nil
 }
 
-func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThorughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool) error {
+func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, tests *map[string][]string, appContainer, networkName *string, Delay *uint64, buildDelay *time.Duration, passThroughPorts *[]uint, apiTimeout *uint64, globalNoise *models.GlobalNoise, testSetNoise *models.TestsetNoise, coverageReportPath *string, withCoverage *bool, configPath string, ignoreOrdering *bool, passThroughHosts *[]models.Filters) error {
 	configFilePath := filepath.Join(configPath, "keploy-config.yaml")
 	if isExist := utils.CheckFileExists(configFilePath); !isExist {
 		return errFileNotFound
@@ -60,7 +60,7 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te
 	if *appCmd == "" {
 		*appCmd = confTest.Command
 	}
-	for testset, testcases := range confTest.Tests {
+	for testset, testcases := range confTest.SelectedTests {
 		if _, ok := (*tests)[testset]; !ok {
 			(*tests)[testset] = testcases
 		}
@@ -77,9 +77,7 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te
 	if *buildDelay == 30*time.Second && confTest.BuildDelay != 0 {
 		*buildDelay = confTest.BuildDelay
 	}
-	if len(*passThorughPorts) == 0 {
-		*passThorughPorts = confTest.PassThroughPorts
-	}
+
 	if len(*coverageReportPath) == 0 {
 		*coverageReportPath = confTest.CoverageReportPath
 	}
@@ -92,6 +90,15 @@ func (t *Test) getTestConfig(path *string, proxyPort *uint32, appCmd *string, te
 	if !*ignoreOrdering {
 		*ignoreOrdering = confTest.IgnoreOrdering
 	}
+	passThroughPortProvided := len(*passThroughPorts) == 0
+	for _, filter := range confTest.Stubs.Filters {
+		if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided {
+			*passThroughPorts = append(*passThroughPorts, filter.Port)
+		} else {
+			*passThroughHosts = append(*passThroughHosts, filter)
+		}
+	}
+
 	return nil
 }
 
@@ -241,10 +248,12 @@ func (t *Test) GetCmd() *cobra.Command {
 			globalNoise := make(models.GlobalNoise)
 			testsetNoise := make(models.TestsetNoise)
 
-			err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering)
+			passThroughHosts := []models.Filters{}
+
+			err = t.getTestConfig(&path, &proxyPort, &appCmd, &tests, &appContainer, &networkName, &delay, &buildDelay, &ports, &apiTimeout, &globalNoise, &testsetNoise, &coverageReportPath, &withCoverage, configPath, &ignoreOrdering, &passThroughHosts)
 			if err != nil {
 				if err == errFileNotFound {
-					t.logger.Info("continuing without configuration file because file not found")
+					t.logger.Info("Keploy config not found, continuing without configuration")
 				} else {
 					t.logger.Error("", zap.Error(err))
 				}
@@ -389,6 +398,7 @@ func (t *Test) GetCmd() *cobra.Command {
 					WithCoverage:       withCoverage,
 					CoverageReportPath: coverageReportPath,
 					IgnoreOrdering:     ignoreOrdering,
+					PassthroughHosts:   passThroughHosts,
 				}, enableTele)
 			}
 

pkg/graph/serve.go

@@ -202,7 +202,7 @@ func (g *graph) Serve(path string, proxyPort uint32, testReportPath string, Dela
 }
 
 // Gracefully shut down the HTTP server with a timeout
-func (g *graph)stopGraphqlServer(httpSrv *http.Server) {
+func (g *graph) stopGraphqlServer(httpSrv *http.Server) {
 	shutdown := make(chan struct{})
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -211,4 +211,4 @@ func (g *graph)stopGraphqlServer(httpSrv *http.Server) {
 	}
 	// If you have other goroutines that should listen for this, you can use this channel to notify them.
 	close(shutdown)
-}
+}

pkg/hooks/connection/factory.go

@@ -38,7 +38,7 @@ func NewFactory(inactivityThreshold time.Duration, logger *zap.Logger) *Factory
 
 // ProcessActiveTrackers iterates over all connection the trackers and checks if they are complete. If so, it captures the ingress call and
 // deletes the tracker. If the tracker is inactive for a long time, it deletes it.
-func (factory *Factory) ProcessActiveTrackers(db platform.TestCaseDB, ctx context.Context, filters *models.Filters) {
+func (factory *Factory) ProcessActiveTrackers(db platform.TestCaseDB, ctx context.Context, filters *models.TestFilter) {
 	factory.mutex.Lock()
 	defer factory.mutex.Unlock()
 	var trackersToDelete []structs.ConnID
@@ -98,7 +98,7 @@ func (factory *Factory) GetOrCreate(connectionID structs.ConnID) *Tracker {
 	return tracker
 }
 
-func capture(db platform.TestCaseDB, req *http.Request, resp *http.Response, logger *zap.Logger, ctx context.Context, reqTimeTest time.Time, resTimeTest time.Time, filters *models.Filters) {
+func capture(db platform.TestCaseDB, req *http.Request, resp *http.Response, logger *zap.Logger, ctx context.Context, reqTimeTest time.Time, resTimeTest time.Time, filters *models.TestFilter) {
 	reqBody, err := io.ReadAll(req.Body)
 	if err != nil {
 		logger.Error("failed to read the http request body", zap.Error(err))

pkg/hooks/loader.go

@@ -93,7 +93,8 @@ type Hook struct {
 	writev        link.Link
 	writevRet     link.Link
 
-	idc clients.InternalDockerClient
+	idc              clients.InternalDockerClient
+	passThroughHosts models.Stubs
 }
 
 func NewHook(db platform.TestCaseDB, mainRoutineId int, logger *zap.Logger) (*Hook, error) {
@@ -141,6 +142,16 @@ func (h *Hook) GetProxyPort() uint32 {
 	return h.proxyPort
 }
 
+func (h *Hook) GetPassThroughHosts() models.Stubs {
+	return h.passThroughHosts
+}
+
+func (h *Hook) SetPassThroughHosts(passThroughHosts []models.Filters) {
+	h.passThroughHosts = models.Stubs{
+		Filters: passThroughHosts,
+	}
+}
+
 func (h *Hook) AppendMocks(m *models.Mock, ctx context.Context) error {
 	h.mu.Lock()
 	defer h.mu.Unlock()
@@ -509,7 +520,7 @@ func (h *Hook) Stop(forceStop bool) {
 // $BPF_CLANG and $BPF_CFLAGS are set by the Makefile.
 //
 //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -no-global-types -target $TARGET bpf keploy_ebpf.c -- -I./headers -I./headers/$TARGET
-func (h *Hook) LoadHooks(appCmd, appContainer string, pid uint32, ctx context.Context, filters *models.Filters) error {
+func (h *Hook) LoadHooks(appCmd, appContainer string, pid uint32, ctx context.Context, filters *models.TestFilter) error {
 	if err := settings.InitRealTimeOffset(); err != nil {
 		h.logger.Error("failed to fix the BPF clock", zap.Error(err))
 		return err

pkg/models/config.go

@@ -8,41 +8,53 @@ type Config struct {
 }
 
 type Record struct {
-	Path             string        `json:"path" yaml:"path"`
-	Command          string        `json:"command" yaml:"command"`
-	ProxyPort        uint32        `json:"proxyport" yaml:"proxyport"`
-	ContainerName    string        `json:"containerName" yaml:"containerName"`
-	NetworkName      string        `json:"networkName" yaml:"networkName"`
-	Delay            uint64        `json:"delay" yaml:"delay"`
-	BuildDelay       time.Duration `json:"buildDelay" yaml:"buildDelay"`
-	PassThroughPorts []uint        `json:"passThroughPorts" yaml:"passThroughPorts"`
-	Filters          Filters       `json:"filters" yaml:"filters"`
+	Path          string        `json:"path" yaml:"path"`
+	Command       string        `json:"command" yaml:"command"`
+	ProxyPort     uint32        `json:"proxyport" yaml:"proxyport"`
+	ContainerName string        `json:"containerName" yaml:"containerName"`
+	NetworkName   string        `json:"networkName" yaml:"networkName"`
+	Delay         uint64        `json:"delay" yaml:"delay"`
+	BuildDelay    time.Duration `json:"buildDelay" yaml:"buildDelay"`
+	Tests         TestFilter    `json:"tests" yaml:"tests"`
+	Stubs         Stubs         `json:"stubs" yaml:"stubs"`
 }
 
+type TestFilter struct {
+	Filters []Filters `json:"filters" yaml:"filters"`
+}
+
+type Stubs struct {
+	Filters []Filters `json:"filters" yaml:"filters"`
+}
 type Filters struct {
-	ReqHeader  []string            `json:"req_header" yaml:"req_header"`
-	URLMethods map[string][]string `json:"urlMethods" yaml:"urlMethods"`
+	Path       string            `json:"path" yaml:"path"`
+	UrlMethods []string          `json:"urlMethods" yaml:"urlMethods"`
+	Host       string            `json:"host" yaml:"host"`
+	Headers    map[string]string `json:"headers" yaml:"headers"`
+	Port       uint              `json:"ports" yaml:"ports"`
 }
 
-func (filter *Filters) GetKind() string {
-	return "filter"
+func (tests *TestFilter) GetKind() string {
+	return "Tests"
 }
 
 type Test struct {
-	Path               string              `json:"path" yaml:"path"`
-	Command            string              `json:"command" yaml:"command"`
-	ProxyPort          uint32              `json:"proxyport" yaml:"proxyport"`
-	ContainerName      string              `json:"containerName" yaml:"containerName"`
-	NetworkName        string              `json:"networkName" yaml:"networkName"`
-	Tests              map[string][]string `json:"tests" yaml:"tests"`
-	GlobalNoise        Globalnoise         `json:"globalNoise" yaml:"globalNoise"`
-	Delay              uint64              `json:"delay" yaml:"delay"`
-	BuildDelay         time.Duration       `json:"buildDelay" yaml:"buildDelay"`
-	ApiTimeout         uint64              `json:"apiTimeout" yaml:"apiTimeout"`
-	PassThroughPorts   []uint              `json:"passThroughPorts" yaml:"passThroughPorts"`
-	WithCoverage       bool                `json:"withCoverage" yaml:"withCoverage"`             // boolean to capture the coverage in test
-	CoverageReportPath string              `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files
-	IgnoreOrdering     bool                `json:"ignoreOrdering" yaml:"ignoreOrdering"`
+	Path                    string              `json:"path" yaml:"path"`
+	Command                 string              `json:"command" yaml:"command"`
+	ProxyPort               uint32              `json:"proxyport" yaml:"proxyport"`
+	ContainerName           string              `json:"containerName" yaml:"containerName"`
+	NetworkName             string              `json:"networkName" yaml:"networkName"`
+	SelectedTests           map[string][]string `json:"selectedTests" yaml:"selectedTests"`
+	GlobalNoise             Globalnoise         `json:"globalNoise" yaml:"globalNoise"`
+	Delay                   uint64              `json:"delay" yaml:"delay"`
+	BuildDelay              time.Duration       `json:"buildDelay" yaml:"buildDelay"`
+	ApiTimeout              uint64              `json:"apiTimeout" yaml:"apiTimeout"`
+	PassThroughPorts        []uint              `json:"passThroughPorts" yaml:"passThroughPorts"`
+	BypassEndpointsRegistry []string            `json:"bypassEndpointsRegistry" yaml:"bypassEndpointsRegistry"`
+	WithCoverage            bool                `json:"withCoverage" yaml:"withCoverage"`             // boolean to capture the coverage in test
+	CoverageReportPath      string              `json:"coverageReportPath" yaml:"coverageReportPath"` // directory path to store the coverage files
+	IgnoreOrdering          bool                `json:"ignoreOrdering" yaml:"ignoreOrdering"`
+	Stubs                   Stubs               `json:"stubs" yaml:"stubs"`
 }
 
 type Globalnoise struct {

pkg/models/const.go

@@ -16,7 +16,7 @@ const (
 )
 
 var (
-	PassThroughHosts = []string{"dc.services.visualstudio.com"}
+	PassThroughHosts = []string{"^dc\\.services\\.visualstudio\\.com$"}
 )
 
 var orangeColorSGR = []color.Attribute{38, 5, 208}