Add request body patching features

README.md

@@ -3,7 +3,7 @@
 ----
 
 <p align="center">
-  <img height="250" alt="PROTTY" src="assets/logo/logo.png"/>
+  <img height="250" alt="PROTTY" src="https://github.com/mgerasimchuk/protty/raw/master/assets/logo/logo.png"/>
 </p>
 
 ----
@@ -18,28 +18,53 @@ These capabilities make Protty a useful tool for a variety of purposes, such as
 The following command will start a proxy on port 8080, and after starting, all traffic from port 8080 will be redirected to a remote host located at https://example.com
 
 ```shell
-docker run -p8080:80 -e REMOTE_URI=https://example.com:443 mgerasimchuk/protty:v0.1.0
+docker run -p8080:80 -e REMOTE_URI=https://example.com:443 mgerasimchuk/protty:v0.2.0
 ```
 
 ## Running options and runtime configuration
 
 ```
-» ~  docker run -p8080:80 -it mgerasimchuk/protty:v0.1.0 /bin/sh -c 'protty start --help'  
+» ~  docker run -p8080:80 -it mgerasimchuk/protty:v0.2.0 /bin/sh -c 'protty start --help'  
 Start the proxy
 
 Usage:
   protty start [flags]
 
 Examples:
+  # Start the proxy with default values
+  protty start
+  
+  # Start the proxy with specific log level
+  protty start --log-level info
+
+  # Start the proxy with a specific local port
+  protty start --local-port 8080
+  
+  # Start the proxy with a specific remote URI and specific throttle rate limit 
   protty start --remote-uri https://www.githubstatus.com --throttle-rate-limit 2
 
+  # Start the proxy with a specific SED expression for response transformation
+  protty start --transform-response-body-sed 's|old|new|g'
+
+  # Start the proxy with a specific SED expressions pipeline for response transformation
+  protty start --transform-response-body-sed 's|old|new-stage-1|g' --transform-response-body-sed 's|new-stage-1|new-stage-2|g'
+
+  # Start the proxy with a specific SED expressions pipeline for response transformation (configured with env)
+  TRANSFORM_RESPONSE_BODY_SED_0='s|old|new-stage-1|g' TRANSFORM_RESPONSE_BODY_SED_1='s|new-stage-1|new-stage-2|g' protty start
+
+  # Start the proxy with a specific JQ expressions pipeline for response transformation
+  protty start --transform-response-body-jq '.[] | .id'
+
 Flags:
-      --log-level string            On which host, the throttle rate limit should be applied | Env variable alias: LOG_LEVEL | Request header alias: X-PROTTY-LOG-LEVEL (default "debug")
-      --local-port int              Verbosity level (panic, fatal, error, warn, info, debug, trace) | Env variable alias: LOCAL_PORT | Request header alias: X-PROTTY-LOCAL-PORT (default 80)
-      --remote-uri string           Listening port for the proxy | Env variable alias: REMOTE_URI | Request header alias: X-PROTTY-REMOTE-URI (default "https://example.com:443")
-      --throttle-rate-limit float   URI of the remote resource | Env variable alias: THROTTLE_RATE_LIMIT | Request header alias: X-PROTTY-THROTTLE-RATE-LIMIT
-      --throttle-host string        How many requests can be send to the remote resource per second | Env variable alias: THROTTLE_HOST | Request header alias: X-PROTTY-THROTTLE-HOST
-  -h, --help                        help for start
+      --log-level string                          On which host, the throttle rate limit should be applied | Env variable alias: LOG_LEVEL | Request header alias: X-PROTTY-LOG-LEVEL (default "debug")
+      --local-port int                            Verbosity level (panic, fatal, error, warn, info, debug, trace) | Env variable alias: LOCAL_PORT | Request header alias: X-PROTTY-LOCAL-PORT (default 80)
+      --remote-uri string                         Listening port for the proxy | Env variable alias: REMOTE_URI | Request header alias: X-PROTTY-REMOTE-URI (default "https://example.com:443")
+      --throttle-rate-limit float                 How many requests can be send to the remote resource per second | Env variable alias: THROTTLE_RATE_LIMIT | Request header alias: X-PROTTY-THROTTLE-RATE-LIMIT
+      --transform-request-body-sed stringArray    Pipeline of SED expressions for request body transformation | Env variable alias: TRANSFORM_REQUEST_BODY_SED | Request header alias: X-PROTTY-TRANSFORM-REQUEST-BODY-SED
+      --transform-request-body-jq stringArray     Pipeline of JQ expressions for request body transformation | Env variable alias: TRANSFORM_REQUEST_BODY_JQ | Request header alias: X-PROTTY-TRANSFORM-REQUEST-BODY-JQ
+      --transform-response-body-sed stringArray   Pipeline of SED expressions for response body transformation | Env variable alias: TRANSFORM_RESPONSE_BODY_SED | Request header alias: X-PROTTY-TRANSFORM-RESPONSE-BODY-SED
+      --transform-response-body-jq stringArray    Pipeline of JQ expressions for response body transformation | Env variable alias: TRANSFORM_RESPONSE_BODY_JQ | Request header alias: X-PROTTY-TRANSFORM-RESPONSE-BODY-JQ
+  -h, --help                                      help for start
 
 *Use CLI flags, environment variables or request headers to configure settings. The settings will be applied in the following priority: environment variables -> CLI flags -> request headers
 ```

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/rwtodd/Go.Sed v0.0.0-20210816025313-55464686f9ef
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.6.1
+	github.com/thoas/go-funk v0.9.3
 	golang.org/x/time v0.3.0
 )
 

go.sum

@@ -23,15 +23,19 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
+github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/adapter/cli/start.go

@@ -36,6 +36,8 @@ func NewStartCommand(cfg *config.StartCommandConfig, reverseProxySvc *service.Re
 	startCommand.cobraCmd.Flags().IntVar(buildFlagArgs(&cfg.LocalPort))
 	startCommand.cobraCmd.Flags().StringVar(buildFlagArgs(&cfg.RemoteURI))
 	startCommand.cobraCmd.Flags().Float64Var(buildFlagArgs(&cfg.ThrottleRateLimit))
+	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformRequestBodySED))
+	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformRequestBodyJQ))
 	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformResponseBodySED))
 	startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformResponseBodyJQ))
 
@@ -77,16 +79,15 @@ func (c *StartCommand) getExamples() string {
 
   # Start the proxy with a specific SED expression for response transformation
   {{ .Cmd.CommandPath }} --{{ .Cfg.TransformResponseBodySED.GetFlagName }} 's|old|new|g'
-	
+
   # Start the proxy with a specific SED expressions pipeline for response transformation
   {{ .Cmd.CommandPath }} --{{ .Cfg.TransformResponseBodySED.GetFlagName }} 's|old|new-stage-1|g' --{{ .Cfg.TransformResponseBodySED.GetFlagName }} 's|new-stage-1|new-stage-2|g'
 
   # Start the proxy with a specific SED expressions pipeline for response transformation (configured with env)
   {{ .Cfg.TransformResponseBodySED.GetEnvName }}_0='s|old|new-stage-1|g' {{ .Cfg.TransformResponseBodySED.GetEnvName }}_1='s|new-stage-1|new-stage-2|g' {{ .Cmd.CommandPath }}
 
   # Start the proxy with a specific JQ expressions pipeline for response transformation
-  {{ .Cmd.CommandPath }} --{{ .Cfg.TransformResponseBodyJQ.GetFlagName }} '.[] | .id'
-`
+  {{ .Cmd.CommandPath }} --{{ .Cfg.TransformResponseBodyJQ.GetFlagName }} '.[] | .id'`
 
 	t, b := new(template.Template), new(strings.Builder)
 	err := template.Must(t.Parse(textTemplate)).Execute(b, struct {

internal/infrastructure/config/start_command_config.go

@@ -19,8 +19,10 @@ type StartCommandConfig struct {
 	LocalPort                Option[int]      `default:"80" description:"Verbosity level (panic, fatal, error, warn, info, debug, trace)"`
 	RemoteURI                Option[string]   `default:"https://example.com:443" description:"Listening port for the proxy"`
 	ThrottleRateLimit        Option[float64]  `description:"How many requests can be send to the remote resource per second"`
-	TransformResponseBodySED Option[[]string] `description:"Pipeline of SED expressions for response transformation"`
-	TransformResponseBodyJQ  Option[[]string] `description:"Pipeline of JQ expressions for response transformation"`
+	TransformRequestBodySED  Option[[]string] `description:"Pipeline of SED expressions for request body transformation"`
+	TransformRequestBodyJQ   Option[[]string] `description:"Pipeline of JQ expressions for request body transformation"`
+	TransformResponseBodySED Option[[]string] `description:"Pipeline of SED expressions for response body transformation"`
+	TransformResponseBodyJQ  Option[[]string] `description:"Pipeline of JQ expressions for response body transformation"`
 }
 
 func GetStartCommandConfig() *StartCommandConfig {

internal/infrastructure/service/reverse_proxy.go

@@ -50,17 +50,48 @@ func (s *ReverseProxyService) logRequestPayload(req *http.Request) {
 // Serve a reverse proxy for a given url
 func (s *ReverseProxyService) serveReverseProxy(res http.ResponseWriter, req *http.Request) {
 	cfg := s.getOverrideConfig(req)
-
-	host := strings.ReplaceAll(strings.ReplaceAll(cfg.RemoteURI.Value, "https://", ""), "http://", "")
-	req.URL.Host = host
-	req.Host = host
-
 	reverseProxy := s.getReverseProxyByParams(*cfg)
+	s.modifyRequest(*cfg, req)
+	reverseProxy.ServeHTTP(res, req)
+}
 
+func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req *http.Request) {
+	host := strings.ReplaceAll(strings.ReplaceAll(cfg.RemoteURI.Value, "https://", ""), "http://", "")
+	req.Host, req.URL.Host = host, host
 	// Deleting encoding to keep availability for changing response
 	req.Header.Del("Accept-Encoding")
 
-	reverseProxy.ServeHTTP(res, req)
+	sourceRequestBody, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(ioutil.ReadAll), err)
+		return
+	}
+	if err = req.Body.Close(); err != nil {
+		s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(req.Body.Close), err)
+		return
+	}
+
+	modifiedRequestBody := sourceRequestBody
+
+	for _, sedExpr := range cfg.TransformRequestBodySED.Value {
+		modifiedRequestBody, sourceRequestBody, err = util.SED(sedExpr, modifiedRequestBody)
+		if err != nil {
+			s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(util.SED), err)
+			return
+		}
+		s.logger.Debugf("ModifyRequestBody: %s", getChangesLogMessage(sourceRequestBody, modifiedRequestBody, sedExpr, cfg.TransformRequestBodySED))
+	}
+	for _, jqExpr := range cfg.TransformRequestBodyJQ.Value {
+		modifiedRequestBody, sourceRequestBody, err = util.JQ(jqExpr, modifiedRequestBody)
+		if err != nil {
+			s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(util.JQ), err)
+			return
+		}
+		s.logger.Debugf("ModifyRequestBody: %s", getChangesLogMessage(sourceRequestBody, modifiedRequestBody, jqExpr, cfg.TransformRequestBodySED))
+	}
+
+	req.Body = ioutil.NopCloser(bytes.NewBuffer(modifiedRequestBody))
+	req.ContentLength = int64(len(modifiedRequestBody))
 }
 
 func (s *ReverseProxyService) getReverseProxyByParams(cfg config.StartCommandConfig) *httputil.ReverseProxy {
@@ -84,38 +115,42 @@ func (s *ReverseProxyService) getReverseProxyByParams(cfg config.StartCommandCon
 
 func (s *ReverseProxyService) getModifyResponseFunc(cfg config.StartCommandConfig) func(resp *http.Response) error {
 	return func(resp *http.Response) error {
-		sourceResponse, err := ioutil.ReadAll(resp.Body)
+		sourceResponseBody, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(ioutil.ReadAll), err)
 			return nil
 		}
-		resp.Body.Close()
-		modifiedResponse := sourceResponse
+		if err = resp.Body.Close(); err != nil {
+			s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(resp.Body.Close), err)
+			return nil
+		}
+
+		modifiedResponseBody := sourceResponseBody
 
 		for _, sedExpr := range cfg.TransformResponseBodySED.Value {
-			modifiedResponse, sourceResponse, err = util.SED(sedExpr, modifiedResponse)
+			modifiedResponseBody, sourceResponseBody, err = util.SED(sedExpr, modifiedResponseBody)
 			if err != nil {
 				s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(util.SED), err)
 				return nil
 			}
-			s.logger.Debugf("ModifyResponse: %s", getChangesLogMessage(sourceResponse, modifiedResponse, sedExpr, cfg.TransformResponseBodySED))
+			s.logger.Debugf("ModifyResponseBody: %s", getChangesLogMessage(sourceResponseBody, modifiedResponseBody, sedExpr, cfg.TransformResponseBodySED))
 		}
 		for _, jqExpr := range cfg.TransformResponseBodyJQ.Value {
-			modifiedResponse, sourceResponse, err = util.JQ(jqExpr, modifiedResponse)
+			modifiedResponseBody, sourceResponseBody, err = util.JQ(jqExpr, modifiedResponseBody)
 			if err != nil {
 				s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(util.JQ), err)
 				return nil
 			}
-			s.logger.Debugf("ModifyResponse: %s", getChangesLogMessage(sourceResponse, modifiedResponse, jqExpr, cfg.TransformResponseBodyJQ))
+			s.logger.Debugf("ModifyResponseBody: %s", getChangesLogMessage(sourceResponseBody, modifiedResponseBody, jqExpr, cfg.TransformResponseBodyJQ))
 		}
 
 		buf := bytes.NewBufferString("")
-		buf.Write(modifiedResponse)
+		buf.Write(modifiedResponseBody)
 		resp.Body = ioutil.NopCloser(buf)
 
-		resp.Body = ioutil.NopCloser(bytes.NewBuffer(modifiedResponse))
-		resp.Header["Content-Length"] = []string{strconv.Itoa(len(modifiedResponse))}
-		resp.ContentLength = int64(len(modifiedResponse))
+		resp.Body = ioutil.NopCloser(bytes.NewBuffer(modifiedResponseBody))
+		resp.Header["Content-Length"] = []string{strconv.Itoa(len(modifiedResponseBody))}
+		resp.ContentLength = int64(len(modifiedResponseBody))
 		return nil
 	}
 }

pkg/util/jq.go

@@ -9,6 +9,10 @@ import (
 // JQ transform the input by jq expression
 // in the error case returns the original input
 func JQ(jqExpr string, input []byte) ([]byte, []byte, error) {
+	if len(input) == 0 {
+		return input, input, nil
+	}
+
 	query, err := gojq.Parse(jqExpr)
 	if err != nil {
 		return input, input, fmt.Errorf("%s: %w", GetFuncName(gojq.Parse), err)

pkg/util/sed.go

@@ -9,6 +9,9 @@ import (
 // SED replace the input by sed expression
 // in the error case returns the original input
 func SED(sedExpr string, input []byte) ([]byte, []byte, error) {
+	if len(input) == 0 {
+		return input, input, nil
+	}
 	engine, err := sed.New(strings.NewReader(sedExpr))
 	if err != nil {
 		return input, input, fmt.Errorf("%s: %w", GetFuncName(sed.New), err)