refactor:parse and loader

cmd/record.go

@@ -77,15 +77,13 @@ func (t *Record) GetRecordConfig(path *string, proxyPort *uint32, appCmd *string
 	if *buildDelay == 30*time.Second && confRecord.BuildDelay != 0 {
 		*buildDelay = confRecord.BuildDelay
 	}
-	*passThrough = append(*passThrough, confRecord.Stubs.Filters...)
+	passThroughPortProvided := len(*passThroughPorts) == 0
 
-	if len(*passThroughPorts) == 0 {
-		for _, filter := range confRecord.Stubs.Filters {
-			if filter.Port != 0 && filter.Host == "" && filter.Path == "" {
-				*passThroughPorts = append(*passThroughPorts, filter.Port)
-			} else {
-				*passThrough = append(*passThrough, filter)
-			}
+	for _, filter := range confRecord.Stubs.Filters {
+		if filter.Port != 0 && filter.Host == "" && filter.Path == "" && passThroughPortProvided {
+			*passThroughPorts = append(*passThroughPorts, filter.Port)
+		} else {
+			*passThrough = append(*passThrough, filter)
 		}
 	}
 

pkg/hooks/loader.go

@@ -95,7 +95,6 @@ type Hook struct {
 
 	idc              clients.InternalDockerClient
 	passThroughHosts models.Stubs
-	sourcePort       int
 }
 
 func NewHook(db platform.TestCaseDB, mainRoutineId int, logger *zap.Logger) (*Hook, error) {
@@ -153,14 +152,6 @@ func (h *Hook) SetProxyHosts(passThroughHosts []models.Filters) {
 	}
 }
 
-func (h *Hook) GetSourcePort() int {
-	return h.sourcePort
-}
-
-func (h *Hook) SetSourcePort(sourcePort int) {
-	h.sourcePort = sourcePort
-}
-
 func (h *Hook) AppendMocks(m *models.Mock, ctx context.Context) error {
 	h.mu.Lock()
 	defer h.mu.Unlock()

pkg/platform/yaml/yaml.go

@@ -130,22 +130,22 @@ func (ys *Yaml) Write(path, fileName string, docRead platform.KindSpecifier) err
 	return nil
 }
 
-func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, requestMethod models.Method) bool {
+func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string, requestMethod models.Method) (error, bool) {
 	urlMatched := false
 	parsedURL, err := url.Parse(requestUrl)
 	if err != nil {
-		return false
+		return err, false
 	}
 
 	// Check for URL path and method
 	regex, err := regexp.Compile(urlStr)
 	if err != nil {
-		return false
+		return err, false
 	}
 
-	urlMatch := regex.FindStringSubmatch(parsedURL.Path)
+	urlMatch := regex.MatchString(parsedURL.Path)
 
-	if len(urlMatch) > 0 && len(urlStr) != 0 {
+	if urlMatch && len(urlStr) != 0 {
 		urlMatched = true
 	}
 
@@ -158,28 +158,29 @@ func containsMatchingUrl(urlMethods []string, urlStr string, requestUrl string,
 		}
 	}
 
-	return urlMatched
+	return nil, urlMatched
 }
 
-func hasBannedHeaders(object map[string]string, bannedHeaders map[string]string) bool {
+func hasBannedHeaders(object map[string]string, bannedHeaders map[string]string) (error, bool) {
 	for headerName, headerNameValue := range object {
 		for bannedHeaderName, bannedHeaderValue := range bannedHeaders {
-			regex, err := regexp.Compile(bannedHeaderValue)
+			regex, err := regexp.Compile(headerName)
 			if err != nil {
-				continue
+				return err, false
 			}
-			headerNameMatch := regex.FindStringSubmatch(headerNameValue)
+			headerNameMatch := regex.MatchString(bannedHeaderName)
+
 			regex, err = regexp.Compile(bannedHeaderValue)
 			if err != nil {
-				continue
+				return err, false
 			}
-			headerValueMatch := regex.FindStringSubmatch(headerNameValue)
-			if len(headerNameMatch) > 0 || len(headerValueMatch) > 0 || headerName == bannedHeaderName || bannedHeaderValue == headerNameValue {
-				return true
+			headerValueMatch := regex.MatchString(headerNameValue)
+			if headerNameMatch || headerValueMatch {
+				return nil, true
 			}
 		}
 	}
-	return false
+	return nil, false
 }
 
 func (ys *Yaml) WriteTestcase(tcRead platform.KindSpecifier, ctx context.Context, filtersRead platform.KindSpecifier) error {
@@ -193,10 +194,14 @@ func (ys *Yaml) WriteTestcase(tcRead platform.KindSpecifier, ctx context.Context
 
 	if ok {
 		for _, testFilter := range testFilters.Filters {
-			if containsMatchingUrl(testFilter.UrlMethods, testFilter.Path, tc.HttpReq.URL, tc.HttpReq.Method) {
+			if err, containsMatch := containsMatchingUrl(testFilter.UrlMethods, testFilter.Path, tc.HttpReq.URL, tc.HttpReq.Method); err == nil && containsMatch {
 				bypassTestCase = true
-			} else if hasBannedHeaders(tc.HttpReq.Header, testFilter.Headers) {
+			} else if err != nil {
+				return fmt.Errorf("%s failed to check matching url, error %s", Emoji, err.Error())
+			} else if bannerHeaderCheck, hasHeader := hasBannedHeaders(tc.HttpReq.Header, testFilter.Headers); bannerHeaderCheck == nil && hasHeader {
 				bypassTestCase = true
+			} else if bannerHeaderCheck != nil {
+				return fmt.Errorf("%s failed to check banned header, error %s", Emoji, err.Error())
 			}
 		}
 	}

pkg/proxy/integrations/grpcparser/grpc_parser.go

@@ -36,7 +36,7 @@ func (g *GrpcParser) OutgoingType(buffer []byte) bool {
 	return bytes.HasPrefix(buffer[:], []byte("PRI * HTTP/2"))
 }
 
-func (g *GrpcParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) {
+func (g *GrpcParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) {
 	switch models.GetMode() {
 	case models.MODE_RECORD:
 		encodeOutgoingGRPC(requestBuffer, clientConn, destConn, g.hooks, g.logger, ctx)

pkg/proxy/integrations/httpparser/httpparser.go

@@ -32,17 +32,17 @@ type HttpParser struct {
 }
 
 // ProcessOutgoing implements proxy.DepInterface.
-func (http *HttpParser) ProcessOutgoing(request []byte, clientConn, destConn net.Conn, ctx context.Context) {
+func (http *HttpParser) ProcessOutgoing(request []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) {
 	switch models.GetMode() {
 	case models.MODE_RECORD:
-		err := encodeOutgoingHttp(request, clientConn, destConn, http.logger, http.hooks, ctx)
+		err := encodeOutgoingHttp(request, clientConn, destConn, http.logger, http.hooks, ctx, sourcePort)
 		if err != nil {
 			http.logger.Error("failed to encode the http message into the yaml", zap.Error(err))
 			return
 		}
 
 	case models.MODE_TEST:
-		decodeOutgoingHttp(request, clientConn, destConn, http.hooks, http.logger)
+		decodeOutgoingHttp(request, clientConn, destConn, http.hooks, http.logger, sourcePort)
 	default:
 		http.logger.Info("Invalid mode detected while intercepting outgoing http call", zap.Any("mode", models.GetMode()))
 	}
@@ -97,14 +97,14 @@ func mapsHaveSameKeys(map1 map[string]string, map2 map[string][]string) bool {
 func ProcessOutgoingHttp(request []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger, ctx context.Context) {
 	switch models.GetMode() {
 	case models.MODE_RECORD:
-		err := encodeOutgoingHttp(request, clientConn, destConn, logger, h, ctx)
+		err := encodeOutgoingHttp(request, clientConn, destConn, logger, h, ctx, 0)
 		if err != nil {
 			logger.Error("failed to encode the http message into the yaml", zap.Error(err))
 			return
 		}
 
 	case models.MODE_TEST:
-		decodeOutgoingHttp(request, clientConn, destConn, h, logger)
+		decodeOutgoingHttp(request, clientConn, destConn, h, logger, 0)
 	default:
 		logger.Info("Invalid mode detected while intercepting outgoing http call", zap.Any("mode", models.GetMode()))
 	}
@@ -431,7 +431,7 @@ func checkIfGzipped(check io.ReadCloser) (bool, *bufio.Reader) {
 }
 
 // Decodes the mocks in test mode so that they can be sent to the user application.
-func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger) {
+func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *hooks.Hook, logger *zap.Logger, sourcePort int) {
 	//Matching algorithmm
 	//Get the mocks
 	for {
@@ -508,8 +508,8 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *
 				if err != nil {
 					continue
 				}
-				subMatches := regex.FindStringSubmatch(req.URL.String())
-				if len(subMatches) > 0 && host != "" || req.Host == host {
+				matches := regex.MatchString(req.URL.String())
+				if matches && host != "" || req.Host == host {
 					passthroughHost = true
 				}
 
@@ -519,8 +519,8 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *
 				if err != nil {
 					continue
 				}
-				subMatches := regex.FindStringSubmatch(req.URL.String())
-				if h.GetSourcePort() == int(filter.Port) && (len(subMatches) > 0 && filter.Path != "") {
+				matches := regex.MatchString(req.URL.String())
+				if sourcePort == int(filter.Port) && (matches && filter.Path != "") {
 					passthroughHost = true
 				}
 			}
@@ -596,7 +596,7 @@ func decodeOutgoingHttp(requestBuffer []byte, clientConn, destConn net.Conn, h *
 }
 
 // encodeOutgoingHttp function parses the HTTP request and response text messages to capture outgoing network calls as mocks.
-func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *zap.Logger, h *hooks.Hook, ctx context.Context) error {
+func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *zap.Logger, h *hooks.Hook, ctx context.Context, sourcePort int) error {
 	var resp []byte
 	var finalResp []byte
 	var finalReq []byte
@@ -690,7 +690,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z
 					}
 
 					// saving last request/response on this connection.
-					err := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h)
+					err := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h, sourcePort)
 					if err != nil {
 						logger.Error("failed to parse the final http request and response", zap.Error(err))
 						return err
@@ -721,7 +721,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z
 			if err == io.EOF {
 				logger.Debug("connection closed by the server", zap.Error(err))
 				//check if before EOF complete response came, and try to parse it.
-				parseErr := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h)
+				parseErr := ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h, sourcePort)
 				if parseErr != nil {
 					logger.Error("failed to parse the final http request and response", zap.Error(parseErr))
 					return parseErr
@@ -735,7 +735,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z
 
 		logger.Debug("This is the final response: " + string(finalResp))
 
-		err = ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h)
+		err = ParseFinalHttp(finalReq, finalResp, reqTimestampMock, resTimestampcMock, ctx, logger, h, sourcePort)
 		if err != nil {
 			logger.Error("failed to parse the final http request and response", zap.Error(err))
 			return err
@@ -764,7 +764,7 @@ func encodeOutgoingHttp(request []byte, clientConn, destConn net.Conn, logger *z
 }
 
 // ParseFinalHttp is used to parse the final http request and response and save it in a yaml file
-func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTimestampcMock time.Time, ctx context.Context, logger *zap.Logger, h *hooks.Hook) error {
+func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTimestampcMock time.Time, ctx context.Context, logger *zap.Logger, h *hooks.Hook, sourcePort int) error {
 	var req *http.Request
 	// converts the request message buffer to http request
 	req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(finalReq)))
@@ -837,8 +837,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime
 		if err != nil {
 			continue
 		}
-		subMatches := regex.FindStringSubmatch(req.URL.String())
-		if len(subMatches) > 0 && host != "" || req.Host == host {
+		matches := regex.MatchString(req.URL.String())
+		if matches && host != "" || req.Host == host {
 			passthroughHost = true
 		}
 	}
@@ -847,8 +847,8 @@ func ParseFinalHttp(finalReq []byte, finalResp []byte, reqTimestampMock, resTime
 		if err != nil {
 			continue
 		}
-		subMatches := regex.FindStringSubmatch(req.URL.String())
-		if h.GetSourcePort() == int(filter.Port) && (len(subMatches) > 0 && filter.Path != "") {
+		matches := regex.MatchString(req.URL.String())
+		if sourcePort == int(filter.Port) && (matches && filter.Path != "") {
 			passthroughHost = true
 		}
 	}

pkg/proxy/integrations/mongoparser/mongoparser.go

@@ -49,7 +49,7 @@ func (m *MongoParser) OutgoingType(buffer []byte) bool {
 	return int(messageLength) == len(buffer)
 }
 
-func (m *MongoParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) {
+func (m *MongoParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) {
 	switch models.GetMode() {
 	case models.MODE_RECORD:
 		m.logger.Debug("the outgoing mongo in record mode")

pkg/proxy/integrations/mysqlparser/mysqlparser.go

@@ -32,7 +32,7 @@ func (sql *MySqlParser) OutgoingType(buffer []byte) bool {
 	//Returning false here because sql parser is using the ports to check if the packet is mysql or not.
 	return false
 }
-func (sql *MySqlParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) {
+func (sql *MySqlParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) {
 	delay := sql.delay
 	switch models.GetMode() {
 	case models.MODE_RECORD:

pkg/proxy/integrations/postgresParser/postgres_parser.go

@@ -58,7 +58,7 @@ func (p *PostgresParser) OutgoingType(buffer []byte) bool {
 	return version == ProtocolVersion
 }
 
-func (p *PostgresParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context) {
+func (p *PostgresParser) ProcessOutgoing(requestBuffer []byte, clientConn, destConn net.Conn, ctx context.Context, sourcePort int) {
 	switch models.GetMode() {
 	case models.MODE_RECORD:
 		encodePostgresOutgoing(requestBuffer, clientConn, destConn, p.hooks, p.logger, ctx)

pkg/proxy/proxy.go

@@ -49,7 +49,7 @@ var Emoji = "\U0001F430" + " Keploy:"
 
 type DependencyHandler interface {
 	OutgoingType(buffer []byte) bool
-	ProcessOutgoing(buffer []byte, conn net.Conn, dst net.Conn, ctx context.Context)
+	ProcessOutgoing(buffer []byte, conn net.Conn, dst net.Conn, ctx context.Context, sourcePort int)
 }
 
 var ParsersMap = make(map[string]DependencyHandler)
@@ -775,8 +775,6 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con
 	remoteAddr := conn.RemoteAddr().(*net.TCPAddr)
 	sourcePort := remoteAddr.Port
 
-	ps.hook.SetSourcePort(sourcePort)
-
 	ps.logger.Debug("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix()))
 
 	//TODO:  fix this bug, getting source port same as proxy port.
@@ -817,7 +815,7 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con
 				// }
 			}
 		}
-		ParsersMap["mysql"].ProcessOutgoing([]byte{}, conn, dst, ctx)
+		ParsersMap["mysql"].ProcessOutgoing([]byte{}, conn, dst, ctx, sourcePort)
 
 	} else {
 		clientConnId := util.GetNextID()
@@ -914,7 +912,7 @@ func (ps *ProxySet) handleConnection(conn net.Conn, port uint32, ctx context.Con
 		//Checking for all the parsers.
 		for _, parser := range ParsersMap {
 			if parser.OutgoingType(buffer) {
-				parser.ProcessOutgoing(buffer, conn, dst, ctx)
+				parser.ProcessOutgoing(buffer, conn, dst, ctx, sourcePort)
 				genericCheck = false
 			}
 		}

pkg/service/generateConfig/generateConfig.go

@@ -75,51 +75,6 @@ test:
         port: 0
   withCoverage: false
   coverageReportPath: ""
-  # Example on using tests
-  # tests: 
-  #   filters:
-  #    - path: "/user/app"
-  #      urlMethods: ["GET"]
-  #      headers: {
-  #        "^asdf*": "^test"
-  #      }
-  #      host: "dc.services.visualstudio.com"
-  # Example on using stubs
-  # stubs: 
-  #   filters:
-  #    - path: "/user/app"
-  #      port: 8080
-  #    - port: 8081
-  #    - host: "dc.services.visualstudio.com"
-  #    - port: 8081
-  #      host: "dc.services.visualstudio.com"
-  #      path: "/user/app"
-  #
-  # Example on using globalNoise
-  # globalNoise: 
-  #    global:
-  #      body: {
-  #         # to ignore some values for a field, 
-  #         # pass regex patterns to the corresponding array value
-  #         "url": ["https?://\S+", "http://\S+"],
-  #      }
-  #      header: {
-  #         # to ignore the entire field, pass an empty array
-  #         "Date": [],
-  #       }
-  #     # to ignore fields or the corresponding values for a specific test-set,
-  #     # pass the test-set-name as a key to the "test-sets" object and
-  #     # populate the corresponding "body" and "header" objects 
-  #     test-sets:
-  #       test-set-1:
-  #         body: {
-  #           # ignore all the values for the "url" field
-  #           "url": []
-  #         }
-  #         header: { 
-  #           # we can also pass the exact value to ignore for a field
-  #           "User-Agent": ["PostmanRuntime/7.34.0"]
-  #         }
 `
 
 func (g *generatorConfig) GenerateConfig(filePath string) {