diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3f8094c..15b0e8f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,14 +7,18 @@ on: jobs: build: - name: Build runs-on: ubuntu-latest + strategy: + matrix: + go: ['1.15', '1.16', '1.17', '1.18', '1.19', '1.20'] + name: Go ${{ matrix.go }} sample + steps: - - name: Set up Go 1.14 + - name: Set up Go 1.13 uses: actions/setup-go@v1 with: - go-version: 1.14 + go-version: ${{ matrix.go }} id: go - name: Check out code into the Go module directory diff --git a/README.md b/README.md index 9874c27..dc52ee3 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,11 @@ gout 是go写的http 客户端，为提高工作效率而开发 - [Using chunked data format](#Using-chunked-data-format) - [NewWithOpt](#NewWithOpt) - [Insecure skip verify](#insecure-skip-verify) - - [Turn off 3xx status code automatic jump](#Turn-off-3xx-status-code-automatic-jump) - - [NewWithOpt set timeout](#NewWithOpt-set-timeout) + - [Turn off 3xx status code automatic jump](#turn-off-3xx-status-code-automatic-jump) + - [NewWithOpt set timeout](#new-with-opt-set-timeout) + - [NewWithOpt unix sock](#new-with-opt-unix-socket) + - [NewWithOpt proxy](#new-with-opt-proxy) + - [NewWithOpt socks5](#new-with-opt-socks5) - [Global configuration](#Global-configuration) - [set timeout](#set-timeout) - [set debug](#set-debug) @@ -2103,7 +2106,7 @@ func main() { } } ``` -## Turn off 3xx status code automatic jump +## turn off 3xx status code automatic jump golang client库默认遇到301的状态码会自动跳转重新发起新请求, 你希望关闭这种默认形为, 那就使用下面的功能 ```go import ( @@ -2120,7 +2123,7 @@ func main() { } } ``` -## NewWithOpt set timeout +## new with opt set timeout ```gout.WithTimeout``` 为了让大家少用```gout.SetTimeout```而设计 ```go import ( @@ -2138,6 +2141,58 @@ func main() { } ``` +## new with opt unix socket +```gout.WithUnixSocket``` 为了让大家少用```.UnixSocket ```而设计 +```go +import ( + "github.com/guonaihong/gout" +) + +func main() { + // globalWithOpt里面包含连接池, 这是一个全局可复用的对象, 一个进程里面可能只需创建1个, 如果有多个不同的unixsocket，可以创建多个 + globalWithOpt := gout.NewWithOpt(gout.WithUnixSocket("/tmp/test.socket")) + err := globalWithOpt.GET("url").Do() + if err != nil { + fmt.Printf("err = %v\n" ,err) + return + } +} +``` +## new with opt proxy +```gout.WithProxy``` 为了让大家少用```.SetProxy ```而设计 +```go +import ( + "github.com/guonaihong/gout" +) + +func main() { + // globalWithOpt里面包含连接池, 这是一个全局可复用的对象, 一个进程里面可能只需创建1个, 如果有多个不同的proxy，可以创建多个 + globalWithOpt := gout.NewWithOpt(gout.WithProxy("http://127.0.0.1:7000")) + err := globalWithOpt.GET("url").Do() + if err != nil { + fmt.Printf("err = %v\n" ,err) + return + } +} +``` + +## new with opt socks5 +```gout.WithSocks5``` 为了让大家少用```.SetSOCKS5```而设计 +```go +import ( + "github.com/guonaihong/gout" +) + +func main() { + // globalWithOpt里面包含连接池, 这是一个全局可复用的对象, 一个进程里面可能只需创建1个, 如果有多个不同的socks5，可以创建多个 + globalWithOpt := gout.NewWithOpt(gout.WithSocks5("127.0.0.1:7000")) + err := globalWithOpt.GET("url").Do() + if err != nil { + fmt.Printf("err = %v\n" ,err) + return + } +} +``` # Global configuration ## set timeout diff --git a/dataflow/dataflow.go b/dataflow/dataflow.go index d31c621..fa259ff 100644 --- a/dataflow/dataflow.go +++ b/dataflow/dataflow.go @@ -151,7 +151,6 @@ func (df *DataFlow) SetRequest(req *http.Request) *DataFlow { // SetBody set the data to the http body, Support string/bytes/io.Reader func (df *DataFlow) SetBody(obj interface{}) *DataFlow { - df.Req.bodyEncoder = encode.NewBodyEncode(obj) return df } @@ -240,8 +239,9 @@ func (df *DataFlow) getTransport() (*http.Transport, bool) { } // UnixSocket 函数会修改Transport, 请像对待全局变量一样对待UnixSocket +// 对于全局变量的解释可看下面的链接 +// https://github.com/guonaihong/gout/issues/373 func (df *DataFlow) UnixSocket(path string) *DataFlow { - df.initTransport() transport, ok := df.getTransport() @@ -258,6 +258,8 @@ func (df *DataFlow) UnixSocket(path string) *DataFlow { } // SetProxy 函数会修改Transport，请像对待全局变量一样对待SetProxy +// 对于全局变量的解释可看下面的链接 +// https://github.com/guonaihong/gout/issues/373 func (df *DataFlow) SetProxy(proxyURL string) *DataFlow { proxy, err := url.Parse(modifyURL(proxyURL)) if err != nil { @@ -279,6 +281,8 @@ func (df *DataFlow) SetProxy(proxyURL string) *DataFlow { } // SetSOCKS5 函数会修改Transport,请像对待全局变量一样对待SetSOCKS5 +// 对于全局变量的解释可看下面的链接 +// https://github.com/guonaihong/gout/issues/373 func (df *DataFlow) SetSOCKS5(addr string) *DataFlow { dialer, err := proxy.SOCKS5("tcp", addr, nil, proxy.Direct) if err != nil { @@ -331,7 +335,6 @@ func (df *DataFlow) BindJSON(obj interface{}) *DataFlow { df.out.RspBodyType = "json" df.Req.bodyDecoder = append(df.Req.bodyDecoder, decode.NewJSONDecode(obj)) return df - } // BindYAML parse the yaml string in http body to obj. diff --git a/dataflow/req.go b/dataflow/req.go index b47d6ce..6d3f34c 100644 --- a/dataflow/req.go +++ b/dataflow/req.go @@ -51,7 +51,7 @@ type Req struct { callback func(*Context) error - //cookie + // cookie cookies []*http.Cookie ctxIndex int @@ -120,7 +120,6 @@ func (r *Req) addDefDebug() { r.ReqBodyType = "yaml" } } - } func (r *Req) addContextType(req *http.Request) { @@ -138,7 +137,6 @@ func (r *Req) addContextType(req *http.Request) { req.Header.Add("Content-Type", "application/x-yaml") } } - } func (r *Req) selectRequest(body *bytes.Buffer) (req *http.Request, err error) { @@ -336,7 +334,7 @@ func (r *Req) Request() (req *http.Request, err error) { if r.userName != nil && r.password != nil { req.SetBasicAuth(*r.userName, *r.password) } - //运行请求中间件 + // 运行请求中间件 for _, reqModify := range r.reqModify { if err = reqModify.ModifyRequest(req); err != nil { return nil, err @@ -377,7 +375,6 @@ func (r *Req) GetContext() context.Context { // TODO 优化代码，每个decode都有自己的指针偏移直接指向流，减少大body的内存使用 func (r *Req) decodeBody(req *http.Request, resp *http.Response) (err error) { - if r.bodyDecoder != nil { var all []byte if len(r.bodyDecoder) > 1 { @@ -385,7 +382,7 @@ func (r *Req) decodeBody(req *http.Request, resp *http.Response) (err error) { if err != nil { return err } - //已经取走数据，直接关闭body + // 已经取走数据，直接关闭body resp.Body.Close() } @@ -432,7 +429,7 @@ func (r *Req) decode(req *http.Request, resp *http.Response, openDebug bool) (er return err } } - //运行响应中间件。放到debug打印后面，避免混淆请求返回内容 + // 运行响应中间件。放到debug打印后面，避免混淆请求返回内容 for _, modify := range r.responseModify { err = modify.ModifyResponse(resp) if err != nil { @@ -461,7 +458,6 @@ func clearBody(resp *http.Response) error { } func (r *Req) Bind(req *http.Request, resp *http.Response) (err error) { - if err = r.decode(req, resp, r.Setting.Debug); err != nil { return err } @@ -485,7 +481,6 @@ func (r *Req) Bind(req *http.Request, resp *http.Response) (err error) { } return nil - } func (r *Req) Client() *http.Client { @@ -528,11 +523,10 @@ func (r *Req) getReqAndRsp() (req *http.Request, rsp *http.Response, err error) // 如果调用Chunked()接口, 就使用chunked的数据包 r.maybeUseChunked(req) - //resp, err := r.Client().Do(req) - //TODO r.Client() 返回Do接口 + // resp, err := r.Client().Do(req) + // TODO r.Client() 返回Do接口 rsp, err = opt.StartTrace(opt, r.canTrace(), req, r.Client()) return - } // Response 获取原始http.Response数据结构 @@ -544,7 +538,6 @@ func (r *Req) Response() (rsp *http.Response, err error) { // Do Send function func (r *Req) Do() (err error) { - req, resp, err := r.getReqAndRsp() if resp != nil { defer resp.Body.Close() diff --git a/gout_options.go b/gout_options.go index 2d23e05..eed097e 100644 --- a/gout_options.go +++ b/gout_options.go @@ -5,12 +5,14 @@ import ( "net/http" "time" + "github.com/guonaihong/gout/hcutil" "github.com/guonaihong/gout/setting" ) type options struct { hc *http.Client setting.Setting + err error } type Option interface { @@ -72,3 +74,48 @@ func WithTimeout(t time.Duration) Option { func (t *timeout) apply(opts *options) { opts.SetTimeout(time.Duration(*t)) } + +// 5. 设置代理 +type proxy string + +func WithProxy(p string) Option { + return (*proxy)(&p) +} + +func (p *proxy) apply(opts *options) { + if opts.hc == nil { + opts.hc = &http.Client{} + } + + opts.err = hcutil.SetProxy(opts.hc, string(*p)) +} + +// 6. 设置socks5代理 +type socks5 string + +func WithSocks5(s string) Option { + return (*socks5)(&s) +} + +func (s *socks5) apply(opts *options) { + if opts.hc == nil { + opts.hc = &http.Client{} + } + + opts.err = hcutil.SetSOCKS5(opts.hc, string(*s)) +} + +// 7. 设置unix socket +type unixSocket string + +func WithUnixSocket(u string) Option { + return (*unixSocket)(&u) +} + +func (u *unixSocket) apply(opts *options) { + if opts.hc == nil { + opts.hc = &http.Client{} + } + + opts.err = hcutil.UnixSocket(opts.hc, string(*u)) +} diff --git a/hcutil/README.md b/hcutil/README.md new file mode 100644 index 0000000..5b2124b --- /dev/null +++ b/hcutil/README.md @@ -0,0 +1,2 @@ +## +本目录存入http.Client的辅助函数 diff --git a/hcutil/hcutil.go b/hcutil/hcutil.go new file mode 100644 index 0000000..9497c37 --- /dev/null +++ b/hcutil/hcutil.go @@ -0,0 +1,83 @@ +package hcutil + +import ( + "fmt" + "net" + "net/http" + "net/url" + "strings" + + "golang.org/x/net/proxy" +) + +func ModifyURL(url string) string { + if strings.HasPrefix(url, "https://") || strings.HasPrefix(url, "http://") { + return url + } + + if strings.HasPrefix(url, ":") { + return fmt.Sprintf("http://127.0.0.1%s", url) + } + + if strings.HasPrefix(url, "/") { + return fmt.Sprintf("http://127.0.0.1%s", url) + } + + return fmt.Sprintf("http://%s", url) +} + +func SetSOCKS5(c *http.Client, addr string) error { + dialer, err := proxy.SOCKS5("tcp", addr, nil, proxy.Direct) + if err != nil { + return err + } + + if c.Transport == nil { + c.Transport = &http.Transport{} + } + + transport, ok := c.Transport.(*http.Transport) + if !ok { + return fmt.Errorf("SetSOCKS5:not found http.transport:%T", c.Transport) + } + + transport.Dial = dialer.Dial + return nil +} + +func SetProxy(c *http.Client, proxyURL string) error { + proxy, err := url.Parse(ModifyURL(proxyURL)) + if err != nil { + return err + } + + if c.Transport == nil { + c.Transport = &http.Transport{} + } + + transport, ok := c.Transport.(*http.Transport) + if !ok { + return fmt.Errorf("SetProxy:not found http.transport:%T", c.Transport) + } + + transport.Proxy = http.ProxyURL(proxy) + + return nil +} + +func UnixSocket(c *http.Client, path string) error { + if c.Transport == nil { + c.Transport = &http.Transport{} + } + + transport, ok := c.Transport.(*http.Transport) + if !ok { + return fmt.Errorf("UnixSocket:not found http.transport:%T", c.Transport) + } + + transport.Dial = func(proto, addr string) (conn net.Conn, err error) { + return net.Dial("unix", path) + } + + return nil +} diff --git a/hcutil/hcutil_test.go b/hcutil/hcutil_test.go new file mode 100644 index 0000000..d08fd18 --- /dev/null +++ b/hcutil/hcutil_test.go @@ -0,0 +1,145 @@ +package hcutil + +import ( + "context" + "errors" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +func setupUnixSocket(t *testing.T, path string) *http.Server { + router := gin.New() + type testHeader struct { + H1 string `header:"h1"` + H2 string `header:"h2"` + } + + router.POST("/test/unix", func(c *gin.Context) { + tHeader := testHeader{} + err := c.ShouldBindHeader(&tHeader) + + assert.Equal(t, tHeader.H1, "v1") + assert.Equal(t, tHeader.H2, "v2") + assert.NoError(t, err) + + c.String(200, "ok") + }) + + listener, err := net.Listen("unix", path) + assert.NoError(t, err) + + srv := http.Server{Handler: router} + go func() { + // 外层是通过context关闭， 所以这里会返回错误 + assert.Error(t, srv.Serve(listener)) + }() + + return &srv +} + +func setupProxy(t *testing.T) *gin.Engine { + r := gin.New() + + r.GET("/:a", func(c *gin.Context) { + all, err := ioutil.ReadAll(c.Request.Body) + + assert.NoError(t, err) + c.String(200, string(all)) + }) + + return r +} + +type TransportFail struct{} + +func (t *TransportFail) RoundTrip(r *http.Request) (*http.Response, error) { + return nil, errors.New("fail") +} + +func TestProxy(t *testing.T) { + router := setupProxy(t) + ts := httptest.NewServer(http.HandlerFunc(router.ServeHTTP)) + defer ts.Close() + proxyTs := httptest.NewServer(http.HandlerFunc(router.ServeHTTP)) + defer proxyTs.Close() + + var s string + var err error + + c := http.Client{} + err = SetProxy(&c, proxyTs.URL) + assert.NoError(t, err) + + req, err := http.NewRequest("GET", ts.URL+"/login", strings.NewReader(proxyTs.URL)) + assert.NoError(t, err) + resp, err := c.Do(req) + assert.NoError(t, err) + + res, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + + s = string(res) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, s, proxyTs.URL) + + err = SetProxy(&c, "\x7f" /*url.Parse源代码写了遇到\x7f会报错*/) + // test fail + assert.Error(t, err) + + // 错误情况1 + c.Transport = &TransportFail{} + req, err = http.NewRequest("GET", ts.URL+"/login", strings.NewReader(s)) + assert.NoError(t, err) + _, err = c.Do(req) + assert.Error(t, err) +} + +func TestSetSOCKS5(t *testing.T) { + // TODO +} + +func TestUnixSocket(t *testing.T) { + path := "./unix.sock" + defer os.Remove(path) + + ctx, cancel := context.WithCancel(context.Background()) + srv := setupUnixSocket(t, path) + defer func() { + assert.NoError(t, srv.Shutdown(ctx)) + cancel() + }() + + c := http.Client{} + UnixSocket(&c, path) + s := "" + + req, err := http.NewRequest("POST", "http://xxx/test/unix/", nil) + req.Header.Add("h1", "v1") + req.Header.Add("h2", "v2") + + resp, err := c.Do(req) + + assert.NoError(t, err) + all, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + s = string(all) + + // err := New(&c).UnixSocket(path).POST("http://xxx/test/unix/").SetHeader(core.H{"h1": "v1", "h2": "v2"}).BindBody(&s).Do() + assert.NoError(t, err) + assert.Equal(t, s, "ok") + + // 错误情况1 + c.Transport = &TransportFail{} + + resp, err = c.Do(req) + + assert.Error(t, err) +}