Add request URL transformation feature

README.md

@@ -18,13 +18,13 @@ These capabilities make Protty a useful tool for a variety of purposes, such as
 The following command will start a proxy on port 8080, and after starting, all traffic from port 8080 will be redirected to a remote host located at https://example.com
 
 ```shell
-docker run -p8080:80 -e REMOTE_URI=https://example.com:443 mgerasimchuk/protty:v0.3.0
+docker run -p8080:80 -e REMOTE_URI=https://example.com:443 mgerasimchuk/protty:v0.4.0
 ```
 
 ## Running options and runtime configuration
 
 ```
-» ~  docker run -p8080:80 -it mgerasimchuk/protty:v0.3.0 /bin/sh -c 'protty start --help'  
+» ~  docker run -p8080:80 -it mgerasimchuk/protty:v0.4.0 /bin/sh -c 'protty start --help'  
 Start the proxy
 
 Usage:
@@ -63,6 +63,7 @@ Flags:
       --local-port int                            Verbosity level (panic, fatal, error, warn, info, debug, trace) | Env variable alias: LOCAL_PORT | Request header alias: X-PROTTY-LOCAL-PORT (default 80)
       --remote-uri string                         Listening port for the proxy | Env variable alias: REMOTE_URI | Request header alias: X-PROTTY-REMOTE-URI (default "https://example.com:443")
       --throttle-rate-limit float                 How many requests can be send to the remote resource per second | Env variable alias: THROTTLE_RATE_LIMIT | Request header alias: X-PROTTY-THROTTLE-RATE-LIMIT
+      --transform-request-url-sed string          SED expression for request URL transformation | Env variable alias: TRANSFORM_REQUEST_URL_SED | Request header alias: X-PROTTY-TRANSFORM-REQUEST-URL-SED
       --additional-request-headers stringArray    Array of additional request headers in format Header: Value | Env variable alias: ADDITIONAL_REQUEST_HEADERS | Request header alias: X-PROTTY-ADDITIONAL-REQUEST-HEADERS
       --transform-request-body-sed stringArray    Pipeline of SED expressions for request body transformation | Env variable alias: TRANSFORM_REQUEST_BODY_SED | Request header alias: X-PROTTY-TRANSFORM-REQUEST-BODY-SED
       --transform-request-body-jq stringArray     Pipeline of JQ expressions for request body transformation | Env variable alias: TRANSFORM_REQUEST_BODY_JQ | Request header alias: X-PROTTY-TRANSFORM-REQUEST-BODY-JQ

internal/adapter/cli/start.go

@@ -36,6 +36,7 @@ func NewStartCommand(cfg *config.StartCommandConfig, reverseProxySvc *service.Re
 	startCommand.cobraCmd.Flags().IntVar(buildFlagArgs(&cfg.LocalPort))
 	startCommand.cobraCmd.Flags().StringVar(buildFlagArgs(&cfg.RemoteURI))
 	startCommand.cobraCmd.Flags().Float64Var(buildFlagArgs(&cfg.ThrottleRateLimit))
+	startCommand.cobraCmd.Flags().StringVar(buildFlagArgs(&cfg.TransformRequestUrlSED))
 	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.AdditionalRequestHeaders))
 	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformRequestBodySED))
 	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformRequestBodyJQ))

internal/infrastructure/config/start_command_config.go

@@ -19,6 +19,7 @@ type StartCommandConfig struct {
 	LocalPort                 Option[int]      `default:"80" description:"Verbosity level (panic, fatal, error, warn, info, debug, trace)"`
 	RemoteURI                 Option[string]   `default:"https://example.com:443" description:"Listening port for the proxy"`
 	ThrottleRateLimit         Option[float64]  `description:"How many requests can be send to the remote resource per second"`
+	TransformRequestUrlSED    Option[string]   `description:"SED expression for request URL transformation"`
 	AdditionalRequestHeaders  Option[[]string] `description:"Array of additional request headers in format Header: Value"`
 	TransformRequestBodySED   Option[[]string] `description:"Pipeline of SED expressions for request body transformation"`
 	TransformRequestBodyJQ    Option[[]string] `description:"Pipeline of JQ expressions for request body transformation"`

internal/infrastructure/service/reverse_proxy.go

@@ -38,12 +38,12 @@ func (s *ReverseProxyService) Start(cfg *config.StartCommandConfig) error {
 }
 
 func (s *ReverseProxyService) handleRequestAndRedirect(res http.ResponseWriter, req *http.Request) {
-	s.logRequestPayload(req)
 	s.serveReverseProxy(res, req)
+	s.logRequestPayload(req)
 }
 
 func (s *ReverseProxyService) logRequestPayload(req *http.Request) {
-	s.logger.WithField("method", req.Method).WithField("path", req.URL.Path).Infof("Send request to %s", req.URL.Host)
+	s.logger.WithField("method", req.Method).WithField("path", req.URL.Path).Infof("Request have been sent to %s", req.URL.Host)
 	// TODO add tracing log with body and other params like headers
 }
 
@@ -61,6 +61,22 @@ func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req *
 	// Deleting encoding to keep availability for changing response
 	req.Header.Del("Accept-Encoding")
 
+	// Transform request URL
+	if cfg.TransformRequestUrlSED.Value != "" {
+		if modifiedURLRaw, _, err := util.SED(cfg.TransformRequestUrlSED.Value, []byte(req.URL.String())); err == nil {
+			sourceURLRaw := req.URL.String()
+			if modifiedURL, err := url.Parse(strings.Trim(string(modifiedURLRaw), "\n")); err == nil { // TODO remove trim (currently it is a hotfix, cos the util.SED added \n at the end unexpectedly)
+				req.URL = modifiedURL
+				s.logger.Debugf("ModifyRequestURL: %s", getChangesLogMessage([]byte(sourceURLRaw), modifiedURLRaw, cfg.TransformRequestUrlSED.Value, cfg.TransformRequestUrlSED))
+			} else {
+				s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(url.Parse), err)
+				return
+			}
+		} else {
+			s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(util.SED), err)
+		}
+	}
+
 	// Add request headers
 	for _, header := range cfg.AdditionalRequestHeaders.Value {
 		kv := strings.SplitN(header, ": ", 2)
@@ -83,6 +99,7 @@ func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req *
 
 	modifiedRequestBody := sourceRequestBody
 
+	// Transform request body with SED
 	for _, sedExpr := range cfg.TransformRequestBodySED.Value {
 		modifiedRequestBody, sourceRequestBody, err = util.SED(sedExpr, modifiedRequestBody)
 		if err != nil {
@@ -91,6 +108,8 @@ func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req *
 		}
 		s.logger.Debugf("ModifyRequestBody: %s", getChangesLogMessage(sourceRequestBody, modifiedRequestBody, sedExpr, cfg.TransformRequestBodySED))
 	}
+
+	// Transform request body with JQ
 	for _, jqExpr := range cfg.TransformRequestBodyJQ.Value {
 		modifiedRequestBody, sourceRequestBody, err = util.JQ(jqExpr, modifiedRequestBody)
 		if err != nil {
@@ -137,6 +156,7 @@ func (s *ReverseProxyService) getModifyResponseFunc(cfg config.StartCommandConfi
 
 		modifiedResponseBody := sourceResponseBody
 
+		// Transform response body with SED
 		for _, sedExpr := range cfg.TransformResponseBodySED.Value {
 			modifiedResponseBody, sourceResponseBody, err = util.SED(sedExpr, modifiedResponseBody)
 			if err != nil {
@@ -145,6 +165,8 @@ func (s *ReverseProxyService) getModifyResponseFunc(cfg config.StartCommandConfi
 			}
 			s.logger.Debugf("ModifyResponseBody: %s", getChangesLogMessage(sourceResponseBody, modifiedResponseBody, sedExpr, cfg.TransformResponseBodySED))
 		}
+
+		// Transform response body with SED
 		for _, jqExpr := range cfg.TransformResponseBodyJQ.Value {
 			modifiedResponseBody, sourceResponseBody, err = util.JQ(jqExpr, modifiedResponseBody)
 			if err != nil {
@@ -191,10 +213,10 @@ func (s *ReverseProxyService) getOverrideConfig(req *http.Request) *config.Start
 
 func getChangesLogMessage[T config.OptionValueType](source, modified []byte, expr string, o config.Option[T]) string {
 	if string(source) == string(modified) {
-		return fmt.Sprintf("the '%s' %s expression didn't change the response", expr, o.Name)
+		return fmt.Sprintf("the '%s' %s expression didn't change the data", expr, o.Name)
 	} else {
 		// TODO add diff to log
-		return fmt.Sprintf("the '%s' %s expression changed the response. Length difference: %d",
+		return fmt.Sprintf("the '%s' %s expression changed the data. Length difference: %d",
 			expr, o.Name, int(math.Abs(float64(len(source)-len(modified)))))
 	}
 }

pkg/util/jq.go

@@ -9,7 +9,7 @@ import (
 // JQ transform the input by jq expression
 // in the error case returns the original input
 func JQ(jqExpr string, input []byte) ([]byte, []byte, error) {
-	if len(input) == 0 {
+	if len(input) == 0 || len(jqExpr) == 0 {
 		return input, input, nil
 	}
 

pkg/util/sed.go

@@ -9,7 +9,7 @@ import (
 // SED replace the input by sed expression
 // in the error case returns the original input
 func SED(sedExpr string, input []byte) ([]byte, []byte, error) {
-	if len(input) == 0 {
+	if len(input) == 0 || len(sedExpr) == 0 {
 		return input, input, nil
 	}
 	engine, err := sed.New(strings.NewReader(sedExpr))