From 19b2f267f24bbccca071f0dd0cadc012957ee570 Mon Sep 17 00:00:00 2001 From: LeeGene <1670044143@qq.com> Date: Fri, 12 May 2023 22:29:28 +0800 Subject: [PATCH] =?UTF-8?q?1.=E4=BF=AE=E5=A4=8DJA3=E6=A8=A1=E6=8B=9F=202.?= =?UTF-8?q?=E6=94=AF=E6=8C=81TLS=E6=8C=87=E7=BA=B9=E4=BF=AE=E6=94=B9=203.?= =?UTF-8?q?=20=E6=94=AF=E6=8C=81HTTP2=E6=8C=87=E7=BA=B9=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=204.=E6=94=AF=E6=8C=81socks5=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 240 +++++++++++++++++- examples/custom_settings.go | 177 +++++++++++++ examples/http2settings-test.go | 126 +++++++++ examples/ja3-test.go | 6 +- examples/tlsextensions-test.go | 91 +++++++ go.mod | 21 +- go.sum | 78 ++---- models/PrepareRequest.go | 12 +- models/Request.go | 20 +- models/Response.go | 2 +- sessions.go | 120 +++++---- transport/client.go | 80 ++++++ {ja3 => transport}/connect.go | 59 +++-- transport/errors.go | 81 ++++++ transport/extensions.go | 159 ++++++++++++ transport/h2settings.go | 110 ++++++++ ja3/transport.go => transport/roundtripper.go | 130 +++++----- {ja3 => transport}/utils.go | 101 ++++---- url/Cookies.go | 4 +- url/Headers.go | 2 +- url/Request.go | 8 +- version.go | 6 +- 22 files changed, 1366 insertions(+), 267 deletions(-) create mode 100644 examples/custom_settings.go create mode 100644 examples/http2settings-test.go create mode 100644 examples/tlsextensions-test.go create mode 100644 transport/client.go rename {ja3 => transport}/connect.go (83%) create mode 100644 transport/errors.go create mode 100644 transport/extensions.go create mode 100644 transport/h2settings.go rename ja3/transport.go => transport/roundtripper.go (61%) rename {ja3 => transport}/utils.go (74%) diff --git a/README.md b/README.md index 432ee94..7adf4df 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ # requests -[![Gitee link address](https://img.shields.io/badge/gitee-reference-red?logo=gitee&logoColor=red&labelColor=white)](https://gitee.com/leegene/requests)[![Github link address](https://img.shields.io/badge/github-reference-blue?logo=github&logoColor=black&labelColor=white&color=black)](https://github.com/wangluozhe/requests)[![Go Version](https://img.shields.io/badge/Go%20Version-1.15.6-blue?logo=go&logoColor=white&labelColor=gray)]()[![Release Version](https://img.shields.io/badge/release-v1.0.42-blue)]()[![go documentation](https://img.shields.io/badge/go-documentation-blue)](https://pkg.go.dev/github.com/wangluozhe/requests)[![license GPL-3.0](https://img.shields.io/badge/license-GPL3.0-orange)](https://github.com/wangluozhe/requests/blob/main/LICENSE) +[![Gitee link address](https://img.shields.io/badge/gitee-reference-red?logo=gitee&logoColor=red&labelColor=white)](https://gitee.com/leegene/requests)[![Github link address](https://img.shields.io/badge/github-reference-blue?logo=github&logoColor=black&labelColor=white&color=black)](https://github.com/wangluozhe/requests)[![Go Version](https://img.shields.io/badge/Go%20Version-1.20-blue?logo=go&logoColor=white&labelColor=gray)]()[![Release Version](https://img.shields.io/badge/release-v1.1.0-blue)]()[![go documentation](https://img.shields.io/badge/go-documentation-blue)](https://pkg.go.dev/github.com/wangluozhe/requests)[![license GPL-3.0](https://img.shields.io/badge/license-GPL3.0-orange)](https://github.com/wangluozhe/requests/blob/main/LICENSE) requests支持以下新特性： 1. 支持http2，默认以http2进行连接，连接失败后会进行退化而进行http1.1连接 2. 支持JA3指纹修改 3. 支持http2+JA3指纹 -4. 支持在使用代理的基础上修改JA3指纹 +4. 支持修改TLS指纹 +5. 支持修改http2指纹 **此模块参考于Python的[requests模块](https://github.com/psf/requests/tree/main/requests)** @@ -23,7 +24,7 @@ go get github.com/wangluozhe/requests ## 下载指定版 ```bash -go get github.com/wangluozhe/requests@v1.0.42 +go get github.com/wangluozhe/requests@v1.1.0 ``` @@ -878,9 +879,240 @@ if err != nil { } fmt.Println(r.Text) -{"ja3_hash":"b32309a26951912be7dba376398abc3b", "ja3": "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"} +{"ja3_hash":"b32309a26951912be7dba376398abc3b", "transport": "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"} ``` +## TLS指纹 + +requests支持你随意修改TLS指纹信息 +```go +package main + +import ( + "fmt" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/requests" + "github.com/wangluozhe/requests/transport" + "github.com/wangluozhe/requests/url" +) + +func main() { + req := url.NewRequest() + headers := &http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0"}, + "accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"}, + "accept-language": []string{"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"}, + "accept-encoding": []string{"gzip, deflate, br"}, + "upgrade-insecure-requests": []string{"1"}, + "sec-fetch-dest": []string{"document"}, + "sec-fetch-mode": []string{"navigate"}, + "sec-fetch-site": []string{"none"}, + "sec-fetch-user": []string{"?1"}, + "te": []string{"trailers"}, + http.PHeaderOrderKey: []string{ + ":method", + ":path", + ":authority", + ":scheme", + }, + http.HeaderOrderKey: []string{ + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "upgrade-insecure-requests", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "te", + }, + } + req.Headers = headers + req.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0" + es := &transport.Extensions{ + SupportedSignatureAlgorithms: []string{ + "ECDSAWithP256AndSHA256", + "ECDSAWithP384AndSHA384", + "ECDSAWithP521AndSHA512", + "PSSWithSHA256", + "PSSWithSHA384", + "PSSWithSHA512", + "PKCS1WithSHA256", + "PKCS1WithSHA384", + "PKCS1WithSHA512", + "ECDSAWithSHA1", + "PKCS1WithSHA1", + }, + //CertCompressionAlgo: []string{ + // "brotli", + //}, + RecordSizeLimit: 4001, + DelegatedCredentials: []string{ + "ECDSAWithP256AndSHA256", + "ECDSAWithP384AndSHA384", + "ECDSAWithP521AndSHA512", + "ECDSAWithSHA1", + }, + SupportedVersions: []string{ + "1.3", + "1.2", + }, + PSKKeyExchangeModes: []string{ + "PskModeDHE", + }, + KeyShareCurves: []string{ + "X25519", + "P256", + }, + } + tes := transport.ToTLSExtensions(es) + req.TLSExtensions = tes + r, err := requests.Get("https://tls.peet.ws/api/all", req) + if err != nil { + fmt.Println(err) + } + fmt.Println(r.Request.Headers) + fmt.Println("url:", r.Url) + fmt.Println("headers:", r.Headers) + fmt.Println("text:", r.Text) +} + +``` + +## HTTP2指纹 + +requests支持HTTP2指纹信息的修改 + +```go +package main + +import ( + "fmt" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/requests" + "github.com/wangluozhe/requests/transport" + "github.com/wangluozhe/requests/url" +) + +func main() { + req := url.NewRequest() + headers := &http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0"}, + "accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"}, + "accept-language": []string{"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"}, + "accept-encoding": []string{"gzip, deflate, br"}, + "upgrade-insecure-requests": []string{"1"}, + "sec-fetch-dest": []string{"document"}, + "sec-fetch-mode": []string{"navigate"}, + "sec-fetch-site": []string{"none"}, + "sec-fetch-user": []string{"?1"}, + "te": []string{"trailers"}, + http.PHeaderOrderKey: []string{ + ":method", + ":path", + ":authority", + ":scheme", + }, + http.HeaderOrderKey: []string{ + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "upgrade-insecure-requests", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "te", + }, + } + req.Headers = headers + req.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0" + h2s := &transport.H2Settings{ + Settings: map[string]int{ + "HEADER_TABLE_SIZE": 65536, + //"ENABLE_PUSH": 0, + //"MAX_HEADER_LIST_SIZE": 262144, + //"MAX_CONCURRENT_STREAMS": 1000, + "INITIAL_WINDOW_SIZE": 131072, + "MAX_FRAME_SIZE": 16384, + }, + SettingsOrder: []string{ + "HEADER_TABLE_SIZE", + "INITIAL_WINDOW_SIZE", + "MAX_FRAME_SIZE", + }, + ConnectionFlow: 12517377, + HeaderPriority: map[string]interface{}{ + "weight": 42, + "streamDep": 13, + "exclusive": false, + }, + PriorityFrames: []map[string]interface{}{ + { + "streamID": 3, + "priorityParam": map[string]interface{}{ + "weight": 201, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 5, + "priorityParam": map[string]interface{}{ + "weight": 101, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 7, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 9, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 7, + "exclusive": false, + }, + }, + { + "streamID": 11, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 3, + "exclusive": false, + }, + }, + { + "streamID": 13, + "priorityParam": map[string]interface{}{ + "weight": 241, + "streamDep": 0, + "exclusive": false, + }, + }, + }, + } + h2ss := transport.ToHTTP2Settings(h2s) + req.HTTP2Settings = h2ss + r, err := requests.Get("https://tls.peet.ws/api/all", req) + if err != nil { + fmt.Println(err) + } + fmt.Println(r.Request.Headers) + fmt.Println("url:", r.Url) + fmt.Println("headers:", r.Headers) + fmt.Println("text:", r.Text) +} + +``` # 编码 diff --git a/examples/custom_settings.go b/examples/custom_settings.go new file mode 100644 index 0000000..669187c --- /dev/null +++ b/examples/custom_settings.go @@ -0,0 +1,177 @@ +package main + +import ( + "fmt" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/requests/transport" + "io" + "log" + "strings" +) + +func main() { + url := "https://tls.peet.ws/api/all" + headers := http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0"}, + "accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"}, + "accept-language": []string{"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"}, + "accept-encoding": []string{"gzip, deflate, br"}, + "upgrade-insecure-requests": []string{"1"}, + "sec-fetch-dest": []string{"document"}, + "sec-fetch-mode": []string{"navigate"}, + "sec-fetch-site": []string{"none"}, + "sec-fetch-user": []string{"?1"}, + "te": []string{"trailers"}, + http.PHeaderOrderKey: []string{ + ":method", + ":path", + ":authority", + ":scheme", + }, + http.HeaderOrderKey: []string{ + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "upgrade-insecure-requests", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "te", + }, + } + var browser = transport.Browser{ + JA3: "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28-21,29-23-24-25-256-257,0", + UserAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0", + } + + es := &transport.Extensions{ + SupportedSignatureAlgorithms: []string{ + "ECDSAWithP256AndSHA256", + "ECDSAWithP384AndSHA384", + "ECDSAWithP521AndSHA512", + "PSSWithSHA256", + "PSSWithSHA384", + "PSSWithSHA512", + "PKCS1WithSHA256", + "PKCS1WithSHA384", + "PKCS1WithSHA512", + "ECDSAWithSHA1", + "PKCS1WithSHA1", + }, + //CertCompressionAlgo: []string{ + // "brotli", + //}, + RecordSizeLimit: 4001, + DelegatedCredentials: []string{ + "ECDSAWithP256AndSHA256", + "ECDSAWithP384AndSHA384", + "ECDSAWithP521AndSHA512", + "ECDSAWithSHA1", + }, + SupportedVersions: []string{ + "1.3", + "1.2", + }, + PSKKeyExchangeModes: []string{ + "PskModeDHE", + }, + KeyShareCurves: []string{ + "X25519", + "P256", + }, + } + h2s := &transport.H2Settings{ + Settings: map[string]int{ + "HEADER_TABLE_SIZE": 65536, + //"ENABLE_PUSH": 0, + //"MAX_HEADER_LIST_SIZE": 262144, + //"MAX_CONCURRENT_STREAMS": 1000, + "INITIAL_WINDOW_SIZE": 131072, + "MAX_FRAME_SIZE": 16384, + }, + SettingsOrder: []string{ + "HEADER_TABLE_SIZE", + "INITIAL_WINDOW_SIZE", + "MAX_FRAME_SIZE", + }, + ConnectionFlow: 12517377, + HeaderPriority: map[string]interface{}{ + "weight": 42, + "streamDep": 13, + "exclusive": false, + }, + PriorityFrames: []map[string]interface{}{ + { + "streamID": 3, + "priorityParam": map[string]interface{}{ + "weight": 201, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 5, + "priorityParam": map[string]interface{}{ + "weight": 101, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 7, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 9, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 7, + "exclusive": false, + }, + }, + { + "streamID": 11, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 3, + "exclusive": false, + }, + }, + { + "streamID": 13, + "priorityParam": map[string]interface{}{ + "weight": 241, + "streamDep": 0, + "exclusive": false, + }, + }, + }, + } + tes := transport.ToTLSExtensions(es) + h2ss := transport.ToHTTP2Settings(h2s) + options := &transport.Options{ + Browser: browser, + Timeout: 30, + TLSExtensions: tes, + HTTP2Settings: h2ss, + } + client, err := transport.NewClient(options) + request, err := http.NewRequest("GET", url, strings.NewReader("")) + if err != nil { + log.Fatalln(err) + } + request.Header = headers + response, err := client.Do(request) + if err != nil { + log.Print("Request Failed: " + err.Error()) + } + content, _ := io.ReadAll(response.Body) + fmt.Println(string(content)) + fmt.Println(response.Header) +} diff --git a/examples/http2settings-test.go b/examples/http2settings-test.go new file mode 100644 index 0000000..fc6c93d --- /dev/null +++ b/examples/http2settings-test.go @@ -0,0 +1,126 @@ +package main + +import ( + "fmt" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/requests" + "github.com/wangluozhe/requests/transport" + "github.com/wangluozhe/requests/url" +) + +func main() { + req := url.NewRequest() + headers := &http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0"}, + "accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"}, + "accept-language": []string{"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"}, + "accept-encoding": []string{"gzip, deflate, br"}, + "upgrade-insecure-requests": []string{"1"}, + "sec-fetch-dest": []string{"document"}, + "sec-fetch-mode": []string{"navigate"}, + "sec-fetch-site": []string{"none"}, + "sec-fetch-user": []string{"?1"}, + "te": []string{"trailers"}, + http.PHeaderOrderKey: []string{ + ":method", + ":path", + ":authority", + ":scheme", + }, + http.HeaderOrderKey: []string{ + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "upgrade-insecure-requests", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "te", + }, + } + req.Headers = headers + req.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0" + h2s := &transport.H2Settings{ + Settings: map[string]int{ + "HEADER_TABLE_SIZE": 65536, + //"ENABLE_PUSH": 0, + //"MAX_HEADER_LIST_SIZE": 262144, + //"MAX_CONCURRENT_STREAMS": 1000, + "INITIAL_WINDOW_SIZE": 131072, + "MAX_FRAME_SIZE": 16384, + }, + SettingsOrder: []string{ + "HEADER_TABLE_SIZE", + "INITIAL_WINDOW_SIZE", + "MAX_FRAME_SIZE", + }, + ConnectionFlow: 12517377, + HeaderPriority: map[string]interface{}{ + "weight": 42, + "streamDep": 13, + "exclusive": false, + }, + PriorityFrames: []map[string]interface{}{ + { + "streamID": 3, + "priorityParam": map[string]interface{}{ + "weight": 201, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 5, + "priorityParam": map[string]interface{}{ + "weight": 101, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 7, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 0, + "exclusive": false, + }, + }, + { + "streamID": 9, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 7, + "exclusive": false, + }, + }, + { + "streamID": 11, + "priorityParam": map[string]interface{}{ + "weight": 1, + "streamDep": 3, + "exclusive": false, + }, + }, + { + "streamID": 13, + "priorityParam": map[string]interface{}{ + "weight": 241, + "streamDep": 0, + "exclusive": false, + }, + }, + }, + } + h2ss := transport.ToHTTP2Settings(h2s) + req.HTTP2Settings = h2ss + r, err := requests.Get("https://tls.peet.ws/api/all", req) + if err != nil { + fmt.Println(err) + } + fmt.Println(r.Request.Headers) + fmt.Println("url:", r.Url) + fmt.Println("headers:", r.Headers) + fmt.Println("text:", r.Text) +} diff --git a/examples/ja3-test.go b/examples/ja3-test.go index 433a299..5f4ff0b 100644 --- a/examples/ja3-test.go +++ b/examples/ja3-test.go @@ -11,9 +11,9 @@ func main() { headers := url.NewHeaders() headers.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") req.Headers = headers - req.Proxies = "http://127.0.0.1:7890" + //req.Proxies = "http://127.0.0.1:8888" req.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0" - r, err := requests.Get("https://ja3er.com/json", req) + r, err := requests.Get("https://tls.peet.ws/api/all", req) if err != nil { fmt.Println(err) } @@ -21,4 +21,4 @@ func main() { fmt.Println("url:", r.Url) fmt.Println("headers:", r.Headers) fmt.Println("text:", r.Text) -} \ No newline at end of file +} diff --git a/examples/tlsextensions-test.go b/examples/tlsextensions-test.go new file mode 100644 index 0000000..b719a2e --- /dev/null +++ b/examples/tlsextensions-test.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/requests" + "github.com/wangluozhe/requests/transport" + "github.com/wangluozhe/requests/url" +) + +func main() { + req := url.NewRequest() + headers := &http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0"}, + "accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"}, + "accept-language": []string{"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"}, + "accept-encoding": []string{"gzip, deflate, br"}, + "upgrade-insecure-requests": []string{"1"}, + "sec-fetch-dest": []string{"document"}, + "sec-fetch-mode": []string{"navigate"}, + "sec-fetch-site": []string{"none"}, + "sec-fetch-user": []string{"?1"}, + "te": []string{"trailers"}, + http.PHeaderOrderKey: []string{ + ":method", + ":path", + ":authority", + ":scheme", + }, + http.HeaderOrderKey: []string{ + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "upgrade-insecure-requests", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "te", + }, + } + req.Headers = headers + req.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0" + es := &transport.Extensions{ + SupportedSignatureAlgorithms: []string{ + "ECDSAWithP256AndSHA256", + "ECDSAWithP384AndSHA384", + "ECDSAWithP521AndSHA512", + "PSSWithSHA256", + "PSSWithSHA384", + "PSSWithSHA512", + "PKCS1WithSHA256", + "PKCS1WithSHA384", + "PKCS1WithSHA512", + "ECDSAWithSHA1", + "PKCS1WithSHA1", + }, + //CertCompressionAlgo: []string{ + // "brotli", + //}, + RecordSizeLimit: 4001, + DelegatedCredentials: []string{ + "ECDSAWithP256AndSHA256", + "ECDSAWithP384AndSHA384", + "ECDSAWithP521AndSHA512", + "ECDSAWithSHA1", + }, + SupportedVersions: []string{ + "1.3", + "1.2", + }, + PSKKeyExchangeModes: []string{ + "PskModeDHE", + }, + KeyShareCurves: []string{ + "X25519", + "P256", + }, + } + tes := transport.ToTLSExtensions(es) + req.TLSExtensions = tes + r, err := requests.Get("https://tls.peet.ws/api/all", req) + if err != nil { + fmt.Println(err) + } + fmt.Println(r.Request.Headers) + fmt.Println("url:", r.Url) + fmt.Println("headers:", r.Headers) + fmt.Println("text:", r.Text) +} diff --git a/go.mod b/go.mod index 76c6f64..64ef6d0 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,21 @@ module github.com/wangluozhe/requests -go 1.15 +go 1.20 require ( - github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 - github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e - github.com/andybalholm/brotli v1.0.4 + github.com/andybalholm/brotli v1.0.5 github.com/bitly/go-simplejson v0.5.0 + github.com/refraction-networking/utls v1.3.2 + github.com/wangluozhe/fhttp v0.0.0-20230512135433-5c2ebfb4868a + golang.org/x/crypto v0.9.0 + golang.org/x/net v0.10.0 +) + +require ( github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/kr/pretty v0.3.0 // indirect - golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 - golang.org/x/net v0.0.0-20220412020605-290c469a71a5 + github.com/gaukas/godicttls v0.0.3 // indirect + github.com/klauspost/compress v1.15.15 // indirect + github.com/kr/pretty v0.3.1 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect ) diff --git a/go.sum b/go.sum index 4ff915e..2d2ca52 100644 --- a/go.sum +++ b/go.sum @@ -1,60 +1,32 @@ -github.com/Danny-Dasilva/fhttp v0.0.0-20220418170016-5ea1c560e6a8 h1:K/qCmMCMhnKGS+q2KOSpYO+UpS7xLKlDX+HgX6LbwsM= -github.com/Danny-Dasilva/fhttp v0.0.0-20220418170016-5ea1c560e6a8/go.mod h1:t534vrahRNn9ax1tRiYSUvwJSa9jWaYYgETlfodBPm4= -github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 h1:Wzbitazy0HugGNRACX7ZB1En21LT/TiVF6YbxoTTqN8= -github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6/go.mod h1:2IT2IFG+d+zzFuj3+ksGtVytcCBsF402zMNWHsWhD2U= -github.com/Danny-Dasilva/utls v0.0.0-20220418055514-7c61e0dbb504/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU= -github.com/Danny-Dasilva/utls v0.0.0-20220418175931-f38e470e04f2/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU= -github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e h1:tqiguW0yAcIwQBQtD+d2rjBnboqB7CwG1OZ12F8avX8= -github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e/go.mod h1:ssfbVNUfWJVRfW41RTpedOUlGXSq3J6aLmirUVkDgJk= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/gaukas/godicttls v0.0.3 h1:YNDIf0d9adcxOijiLrEzpfZGAkNwLRzPaG6OjU7EITk= +github.com/gaukas/godicttls v0.0.3/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= +github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/wangluozhe/fhttp v0.0.0-20230512094319-9596880bdf8a h1:BNnudZ48Z56ASQix+6tKUhizpPF3rc/Q+wE7dc98hCM= +github.com/wangluozhe/fhttp v0.0.0-20230512094319-9596880bdf8a/go.mod h1:kAK+x1U0Wmy/htOSEeV31JyFBAVndp/orqVJZTq9FxM= +github.com/wangluozhe/fhttp v0.0.0-20230512135433-5c2ebfb4868a h1:nFqhBDkWfNrI5h8nAOv4orMHi0w3qMrd7GoBFXXZGmc= +github.com/wangluozhe/fhttp v0.0.0-20230512135433-5c2ebfb4868a/go.mod h1:kAK+x1U0Wmy/htOSEeV31JyFBAVndp/orqVJZTq9FxM= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/models/PrepareRequest.go b/models/PrepareRequest.go index 5894f07..673abe7 100644 --- a/models/PrepareRequest.go +++ b/models/PrepareRequest.go @@ -6,8 +6,8 @@ import ( jsonp "encoding/json" "errors" "fmt" - "github.com/Danny-Dasilva/fhttp" - "github.com/Danny-Dasilva/fhttp/cookiejar" + "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/fhttp/cookiejar" "github.com/wangluozhe/requests/url" "github.com/wangluozhe/requests/utils" "io" @@ -39,7 +39,7 @@ type PrepareRequest struct { Url string Headers *http.Header Cookies *cookiejar.Jar - Body io.ReadCloser + Body io.Reader } // 预处理所有数据 @@ -140,7 +140,7 @@ func (pr *PrepareRequest) Prepare_body(data *url.Values, files *url.Files, json } if files != nil { var byteBuffer *bytes.Buffer - if data != nil{ + if data != nil { for _, key := range data.Keys() { files.AddField(key, data.Get(key)) } @@ -162,7 +162,7 @@ func (pr *PrepareRequest) Prepare_body(data *url.Values, files *url.Files, json if content_type != "" && pr.Headers.Get("Content-Type") == "" { pr.Headers.Set("Content-Type", content_type) } - pr.Body = ioutil.NopCloser(strings.NewReader(body)) + pr.Body = strings.NewReader(body) return nil } @@ -204,7 +204,7 @@ func (pr *PrepareRequest) Prepare_auth(auth []string, rawurl string) error { } } if auth != nil && len(auth) == 2 { - pr.Headers.Set("Authorization", "Basic " + base64.StdEncoding.EncodeToString([]byte(strings.Join(auth, ":")))) + pr.Headers.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(strings.Join(auth, ":")))) } return nil } diff --git a/models/Request.go b/models/Request.go index bee7135..f962af4 100644 --- a/models/Request.go +++ b/models/Request.go @@ -1,21 +1,21 @@ package models import ( - "github.com/Danny-Dasilva/fhttp" - "github.com/Danny-Dasilva/fhttp/cookiejar" + "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/fhttp/cookiejar" "github.com/wangluozhe/requests/url" ) type Request struct { - Method string - Url string - Params *url.Params + Method string + Url string + Params *url.Params Headers *http.Header Cookies *cookiejar.Jar - Data *url.Values - Files *url.Files - Json map[string]interface{} - Auth []string + Data *url.Values + Files *url.Files + Json map[string]interface{} + Auth []string } func (req *Request) Prepare() *PrepareRequest { @@ -32,4 +32,4 @@ func (req *Request) Prepare() *PrepareRequest { req.Auth, ) return p -} \ No newline at end of file +} diff --git a/models/Response.go b/models/Response.go index c6be86c..3567c42 100644 --- a/models/Response.go +++ b/models/Response.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "github.com/Danny-Dasilva/fhttp" "github.com/bitly/go-simplejson" + "github.com/wangluozhe/fhttp" "github.com/wangluozhe/requests/url" "io" ) diff --git a/sessions.go b/sessions.go index ffaa531..f5bbd61 100644 --- a/sessions.go +++ b/sessions.go @@ -8,15 +8,17 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/Danny-Dasilva/fhttp" - "github.com/Danny-Dasilva/fhttp/cookiejar" - tls "github.com/Danny-Dasilva/utls" "github.com/andybalholm/brotli" - "github.com/wangluozhe/requests/ja3" + tls "github.com/refraction-networking/utls" + "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/fhttp/cookiejar" + "github.com/wangluozhe/fhttp/http2" "github.com/wangluozhe/requests/models" + ja3 "github.com/wangluozhe/requests/transport" "github.com/wangluozhe/requests/url" "github.com/wangluozhe/requests/utils" "io/ioutil" + "log" url2 "net/url" "strings" "time" @@ -69,6 +71,9 @@ func merge_setting(request_setting, session_setting interface{}) interface{} { return merged_setting } for key, _ := range *requestd_setting { + if key == http.PHeaderOrderKey || key == http.HeaderOrderKey { + continue + } merged_setting.Set(key, (*requestd_setting)[key][0]) } return merged_setting @@ -101,6 +106,24 @@ func merge_setting(request_setting, session_setting interface{}) interface{} { if requestd_setting == "" { return merged_setting } + case *ja3.TLSExtensions: + merged_setting := session_setting.(*ja3.TLSExtensions) + if merged_setting == nil { + return request_setting + } + requestd_setting := request_setting.(*ja3.TLSExtensions) + if requestd_setting == nil { + return merged_setting + } + case *http2.HTTP2Settings: + merged_setting := session_setting.(*http2.HTTP2Settings) + if merged_setting == nil { + return request_setting + } + requestd_setting := request_setting.(*http2.HTTP2Settings) + if requestd_setting == nil { + return merged_setting + } } return request_setting } @@ -132,7 +155,7 @@ func NewSession() *Session { TLSClientConfig: &tls.Config{ InsecureSkipVerify: session.Verify, }, - DisableKeepAlives: false, // 这里问题很严重 + DisableKeepAlives: false, // 这里问题很严重 } session.request = &http.Request{} session.client = &http.Client{ @@ -151,18 +174,20 @@ func DefaultSession() *Session { // Session结构体 type Session struct { - Params *url.Params - Headers *http.Header - Cookies *cookiejar.Jar - Auth []string - Proxies string - Verify bool - Cert []string - Ja3 string - MaxRedirects int - transport *http.Transport - request *http.Request - client *http.Client + Params *url.Params + Headers *http.Header + Cookies *cookiejar.Jar + Auth []string + Proxies string + Verify bool + Cert []string + Ja3 string + MaxRedirects int + TLSExtensions *ja3.TLSExtensions + HTTP2Settings *http2.HTTP2Settings + transport *http.Transport + request *http.Request + client *http.Client } // 预请求处理 @@ -273,19 +298,36 @@ func (s *Session) Send(preq *models.PrepareRequest, req *url.Request) (*models.R var err error var history []*models.Response - // 修复报错tls: CurvePreferences includes unsupported curve - s.transport.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: s.Verify, - } - // 设置代理 proxies := merge_setting(s.Proxies, req.Proxies).(string) - if proxies != "" { - u1, err := url2.Parse(proxies) + + // 设置JA3指纹信息 + ja3String := merge_setting(s.Ja3, req.Ja3).(string) + if ja3String != "" && strings.HasPrefix(preq.Url, "https") { + browser := ja3.Browser{ + JA3: ja3String, + UserAgent: s.Headers.Get("User-Agent"), + } + + // 自定义TLS指纹信息 + tlsExtensions := merge_setting(req.TLSExtensions, s.TLSExtensions).(*ja3.TLSExtensions) + http2Settings := merge_setting(req.HTTP2Settings, s.HTTP2Settings).(*http2.HTTP2Settings) + + options := &ja3.Options{ + Browser: browser, + TLSExtensions: tlsExtensions, + HTTP2Settings: http2Settings, + } + + if proxies != "" { + options.Proxy = proxies + } + + client, err := ja3.NewClient(options) if err != nil { return nil, err } - s.transport.Proxy = http.ProxyURL(u1) + s.client = &client } // 是否验证证书 @@ -318,20 +360,6 @@ func (s *Session) Send(preq *models.PrepareRequest, req *url.Request) (*models.R s.transport.TLSClientConfig.Certificates = []tls.Certificate{certs} } - // 设置JA3指纹信息 - ja3String := merge_setting(s.Ja3, req.Ja3).(string) - if ja3String != "" && strings.HasPrefix(preq.Url, "https") { - browser := ja3.Browser{ - JA3: ja3String, - UserAgent: s.Headers.Get("User-Agent"), - } - tr, err := ja3.NewJA3Transport(browser, proxies, s.transport.TLSClientConfig) - if err != nil { - return nil, err - } - s.client.Transport = tr - } - // 设置超时时间 timeout := req.Timeout if timeout != 0 { @@ -360,19 +388,21 @@ func (s *Session) Send(preq *models.PrepareRequest, req *url.Request) (*models.R s.client.CheckRedirect = disableRedirect } - u, _ := url2.Parse(utils.EncodeURI(preq.Url)) // 设置有序请求头 if req.Headers != nil { if (*req.Headers)[http.HeaderOrderKey] != nil { (*preq.Headers)[http.HeaderOrderKey] = (*req.Headers)[http.HeaderOrderKey] } + if (*req.Headers)[http.PHeaderOrderKey] != nil { + (*preq.Headers)[http.PHeaderOrderKey] = (*req.Headers)[http.PHeaderOrderKey] + } } - s.request = &http.Request{ - Method: preq.Method, - URL: u, - Header: *preq.Headers, - Body: preq.Body, + + s.request, err = http.NewRequest(preq.Method, preq.Url, preq.Body) + if err != nil { + log.Fatalln(err) } + s.request.Header = *preq.Headers s.client.Jar = preq.Cookies req.Headers = &s.request.Header resp, err := s.client.Do(s.request) diff --git a/transport/client.go b/transport/client.go new file mode 100644 index 0000000..403a1fd --- /dev/null +++ b/transport/client.go @@ -0,0 +1,80 @@ +package transport + +import ( + utls "github.com/refraction-networking/utls" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/fhttp/http2" + + "time" + + "golang.org/x/net/proxy" +) + +type Browser struct { + // Return a greeting that embeds the name in a message. + JA3 string + UserAgent string +} + +var disabledRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse +} + +func clientBuilder(browser Browser, config *utls.Config, tlsExtensions *TLSExtensions, http2Settings *http2.HTTP2Settings, forceHTTP1 bool, dialer proxy.ContextDialer, timeout int) http.Client { + //if timeout is not set in call default to 15 + if timeout == 0 { + timeout = 15 + } + client := http.Client{ + Transport: newRoundTripper(browser, config, tlsExtensions, http2Settings, forceHTTP1, dialer), + Timeout: time.Duration(timeout) * time.Second, + } + return client +} + +// newClient creates a new http transport +func newClient(browser Browser, timeout int, config *utls.Config, tlsExtensions *TLSExtensions, http2Settings *http2.HTTP2Settings, forceHTTP1 bool, proxyURL ...string) (http.Client, error) { + //fix check PR + if len(proxyURL) > 0 && len(proxyURL[0]) > 0 { + dialer, err := newConnectDialer(proxyURL[0], browser.UserAgent) + if err != nil { + return http.Client{ + Timeout: time.Duration(timeout) * time.Second, + }, err + } + return clientBuilder( + browser, + config, + tlsExtensions, + http2Settings, + forceHTTP1, + dialer, + timeout, + ), nil + } + + return clientBuilder( + browser, + config, + tlsExtensions, + http2Settings, + forceHTTP1, + proxy.Direct, + timeout, + ), nil + +} + +type Options struct { + Browser Browser + Timeout int + TLSConfig *utls.Config + TLSExtensions *TLSExtensions + HTTP2Settings *http2.HTTP2Settings + ForceHTTP1 bool + Proxy string +} + +func NewClient(options *Options) (http.Client, error) { + return newClient(options.Browser, options.Timeout, options.TLSConfig, options.TLSExtensions, options.HTTP2Settings, options.ForceHTTP1, options.Proxy) +} diff --git a/ja3/connect.go b/transport/connect.go similarity index 83% rename from ja3/connect.go rename to transport/connect.go index ec2132c..cf597dc 100644 --- a/ja3/connect.go +++ b/transport/connect.go @@ -1,4 +1,4 @@ -package ja3 +package transport // borrowed from from https://github.com/caddyserver/forwardproxy/blob/master/httpclient/httpclient.go import ( @@ -7,23 +7,23 @@ import ( "crypto/tls" "encoding/base64" "errors" + "fmt" + http "github.com/wangluozhe/fhttp" + http2 "github.com/wangluozhe/fhttp/http2" "golang.org/x/net/proxy" "io" "net" "net/url" "strconv" "sync" - - http "github.com/Danny-Dasilva/fhttp" - http2 "github.com/Danny-Dasilva/fhttp/http2" ) -// connectDialer allows to configure one-time use HTTP CONNECT client +// connectDialer allows to configure one-time use HTTP CONNECT transport type connectDialer struct { ProxyURL url.URL DefaultHeader http.Header - Dialer net.Dialer // overridden dialer allow to control establishment of TCP connection + Dialer proxy.ContextDialer // overridden dialer allow to control establishment of TCP connection // overridden DialTLS allows user to control establishment of TLS connection // MUST return connection with completed Handshake, and NegotiatedProtocol @@ -38,7 +38,7 @@ type connectDialer struct { // newConnectDialer creates a dialer to issue CONNECT requests and tunnel traffic via HTTP/S proxy. // proxyUrlStr must provide Scheme and Host, may provide credentials and port. // Example: https://username:password@golang.org:443 -func NewConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer, error) { +func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer, error) { proxyURL, err := url.Parse(proxyURLStr) if err != nil { return nil, err @@ -49,6 +49,12 @@ func NewConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer "`, make sure to specify full url like https://username:password@hostname.com:443/") } + client := &connectDialer{ + ProxyURL: *proxyURL, + DefaultHeader: make(http.Header), + EnableH2ConnReuse: true, + } + switch proxyURL.Scheme { case "http": if proxyURL.Port() == "" { @@ -58,28 +64,44 @@ func NewConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer if proxyURL.Port() == "" { proxyURL.Host = net.JoinHostPort(proxyURL.Host, "443") } + case "socks5": + var auth *proxy.Auth + if proxyURL.User != nil { + if proxyURL.User.Username() != "" { + username := proxyURL.User.Username() + password, _ := proxyURL.User.Password() + auth = &proxy.Auth{User: username, Password: password} + } + } + dialSocksProxy, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, nil) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error creating SOCKS5 proxy, reason %s", err)) + } + if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok { + client.Dialer = contextDialer + } else { + return nil, errors.New("failed type assertion to DialContext") + } + client.DefaultHeader.Set("User-Agent", UserAgent) + return client, nil case "": return nil, errors.New("specify scheme explicitly (https://)") default: return nil, errors.New("scheme " + proxyURL.Scheme + " is not supported") } - client := &connectDialer{ - ProxyURL: *proxyURL, - DefaultHeader: make(http.Header), - EnableH2ConnReuse: true, - } + client.Dialer = &net.Dialer{} if proxyURL.User != nil { if proxyURL.User.Username() != "" { // password, _ := proxyUrl.User.Password() - // client.DefaultHeader.Set("Proxy-Authorization", "Basic "+ + // transport.DefaultHeader.Set("Proxy-Authorization", "Basic "+ // base64.StdEncoding.EncodeToString([]byte(proxyUrl.User.Username()+":"+password))) username := proxyURL.User.Username() password, _ := proxyURL.User.Password() - // client.DefaultHeader.SetBasicAuth(username, password) + // transport.DefaultHeader.SetBasicAuth(username, password) auth := username + ":" + password basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) client.DefaultHeader.Add("Proxy-Authorization", basicAuth) @@ -99,6 +121,10 @@ type ContextKeyHeader struct{} // ctx.Value will be inspected for optional ContextKeyHeader{} key, with `http.Header` value, // which will be added to outgoing request headers, overriding any colliding c.DefaultHeader func (c *connectDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if c.ProxyURL.Scheme == "socks5" { + return c.Dialer.DialContext(ctx, network, address) + } + req := (&http.Request{ Method: "CONNECT", URL: &url.URL{Host: address}, @@ -195,8 +221,9 @@ func (c *connectDialer) DialContext(ctx context.Context, network, address string } } else { tlsConf := tls.Config{ - NextProtos: []string{"h2", "http/1.1"}, - ServerName: c.ProxyURL.Hostname(), + NextProtos: []string{"h2", "http/1.1"}, + ServerName: c.ProxyURL.Hostname(), + InsecureSkipVerify: true, } tlsConn, err := tls.Dial(network, c.ProxyURL.Host, &tlsConf) if err != nil { diff --git a/transport/errors.go b/transport/errors.go new file mode 100644 index 0000000..489d028 --- /dev/null +++ b/transport/errors.go @@ -0,0 +1,81 @@ +package transport + +import ( + "fmt" + "net" + "net/url" + "os" + "strconv" + "strings" +) + +type errorMessage struct { + StatusCode int + debugger string + ErrorMsg string + Op string +} + +func lastString(ss []string) string { + return ss[len(ss)-1] +} + +// func createErrorString(err: string) (msg, debugger string) { +func createErrorString(err error) (msg, debugger string) { + msg = fmt.Sprintf("Request returned a Syscall Error: %s", err) + debugger = fmt.Sprintf("%#v\n", err) + return +} + +func createErrorMessage(StatusCode int, err error, op string) errorMessage { + msg := fmt.Sprintf("Request returned a Syscall Error: %s", err) + debugger := fmt.Sprintf("%#v\n", err) + return errorMessage{StatusCode: StatusCode, debugger: debugger, ErrorMsg: msg, Op: op} +} + +func parseError(err error) (errormessage errorMessage) { + var op string + + httpError := string(err.Error()) + status := lastString(strings.Split(httpError, "StatusCode:")) + StatusCode, _ := strconv.Atoi(status) + if StatusCode != 0 { + msg, debugger := createErrorString(err) + return errorMessage{StatusCode: StatusCode, debugger: debugger, ErrorMsg: msg} + } + if uerr, ok := err.(*url.Error); ok { + if noerr, ok := uerr.Err.(*net.OpError); ok { + op = noerr.Op + if SyscallError, ok := noerr.Err.(*os.SyscallError); ok { + if noerr.Timeout() { + return createErrorMessage(408, SyscallError, op) + } + return createErrorMessage(401, SyscallError, op) + } else if AddrError, ok := noerr.Err.(*net.AddrError); ok { + return createErrorMessage(405, AddrError, op) + } else if DNSError, ok := noerr.Err.(*net.DNSError); ok { + return createErrorMessage(421, DNSError, op) + } else { + return createErrorMessage(421, noerr, op) + } + } + if uerr.Timeout() { + return createErrorMessage(408, uerr, op) + } + } + return +} + +type errExtensionNotExist struct { + Context string +} + +func (w *errExtensionNotExist) Error() string { + return fmt.Sprintf("Extension {{ %s }} is not Supported by requests please raise an issue", w.Context) +} + +func raiseExtensionError(info string) *errExtensionNotExist { + return &errExtensionNotExist{ + Context: info, + } +} diff --git a/transport/extensions.go b/transport/extensions.go new file mode 100644 index 0000000..4c9473f --- /dev/null +++ b/transport/extensions.go @@ -0,0 +1,159 @@ +package transport + +import ( + "fmt" + utls "github.com/refraction-networking/utls" + "strconv" +) + +var supportedSignatureAlgorithmsExtensions = map[string]utls.SignatureScheme{ + "PKCS1WithSHA256": utls.PKCS1WithSHA256, + "PKCS1WithSHA384": utls.PKCS1WithSHA384, + "PKCS1WithSHA512": utls.PKCS1WithSHA512, + "PSSWithSHA256": utls.PSSWithSHA256, + "PSSWithSHA384": utls.PSSWithSHA384, + "PSSWithSHA512": utls.PSSWithSHA512, + "ECDSAWithP256AndSHA256": utls.ECDSAWithP256AndSHA256, + "ECDSAWithP384AndSHA384": utls.ECDSAWithP384AndSHA384, + "ECDSAWithP521AndSHA512": utls.ECDSAWithP521AndSHA512, + "Ed25519": utls.Ed25519, + "PKCS1WithSHA1": utls.PKCS1WithSHA1, + "ECDSAWithSHA1": utls.ECDSAWithSHA1, +} + +var certCompressionAlgoExtensions = map[string]utls.CertCompressionAlgo{ + "zlib": utls.CertCompressionZlib, + "brotli": utls.CertCompressionBrotli, + "zstd": utls.CertCompressionZstd, +} + +var supportedVersionsExtensions = map[string]uint16{ + "GREASE": utls.GREASE_PLACEHOLDER, + "1.3": utls.VersionTLS13, + "1.2": utls.VersionTLS12, + "1.1": utls.VersionTLS11, + "1.0": utls.VersionTLS10, +} + +var pskKeyExchangeModesExtensions = map[string]uint8{ + "PskModeDHE": utls.PskModeDHE, + "PskModePlain": utls.PskModePlain, +} + +var keyShareCurvesExtensions = map[string]utls.KeyShare{ + "GREASE": utls.KeyShare{Group: utls.CurveID(utls.GREASE_PLACEHOLDER), Data: []byte{0}}, + "P256": utls.KeyShare{Group: utls.CurveP256}, + "P384": utls.KeyShare{Group: utls.CurveP384}, + "P521": utls.KeyShare{Group: utls.CurveP521}, + "X25519": utls.KeyShare{Group: utls.X25519}, +} + +type Extensions struct { + //PKCS1WithSHA256 SignatureScheme = 0x0401 + //PKCS1WithSHA384 SignatureScheme = 0x0501 + //PKCS1WithSHA512 SignatureScheme = 0x0601 + //PSSWithSHA256 SignatureScheme = 0x0804 + //PSSWithSHA384 SignatureScheme = 0x0805 + //PSSWithSHA512 SignatureScheme = 0x0806 + //ECDSAWithP256AndSHA256 SignatureScheme = 0x0403 + //ECDSAWithP384AndSHA384 SignatureScheme = 0x0503 + //ECDSAWithP521AndSHA512 SignatureScheme = 0x0603 + //Ed25519 SignatureScheme = 0x0807 + //PKCS1WithSHA1 SignatureScheme = 0x0201 + //ECDSAWithSHA1 SignatureScheme = 0x0203 + SupportedSignatureAlgorithms []string + //CertCompressionZlib CertCompressionAlgo = 0x0001 + //CertCompressionBrotli CertCompressionAlgo = 0x0002 + //CertCompressionZstd CertCompressionAlgo = 0x0003 + CertCompressionAlgo []string + // Limit: 0x4001 + RecordSizeLimit int + //PKCS1WithSHA256 SignatureScheme = 0x0401 + //PKCS1WithSHA384 SignatureScheme = 0x0501 + //PKCS1WithSHA512 SignatureScheme = 0x0601 + //PSSWithSHA256 SignatureScheme = 0x0804 + //PSSWithSHA384 SignatureScheme = 0x0805 + //PSSWithSHA512 SignatureScheme = 0x0806 + //ECDSAWithP256AndSHA256 SignatureScheme = 0x0403 + //ECDSAWithP384AndSHA384 SignatureScheme = 0x0503 + //ECDSAWithP521AndSHA512 SignatureScheme = 0x0603 + //Ed25519 SignatureScheme = 0x0807 + //PKCS1WithSHA1 SignatureScheme = 0x0201 + //ECDSAWithSHA1 SignatureScheme = 0x0203 + DelegatedCredentials []string + //GREASE_PLACEHOLDER = 0x0a0a + //VersionTLS10 = 0x0301 + //VersionTLS11 = 0x0302 + //VersionTLS12 = 0x0303 + //VersionTLS13 = 0x0304 + //VersionSSL30 = 0x0300 + SupportedVersions []string + //PskModePlain uint8 = pskModePlain + //PskModeDHE uint8 = pskModeDHE + PSKKeyExchangeModes []string + //GREASE_PLACEHOLDER = 0x0a0a + //CurveP256 CurveID = 23 + //CurveP384 CurveID = 24 + //CurveP521 CurveID = 25 + //X25519 CurveID = 29 + KeyShareCurves []string +} + +type TLSExtensions struct { + SupportedSignatureAlgorithms *utls.SignatureAlgorithmsExtension + CertCompressionAlgo *utls.UtlsCompressCertExtension + RecordSizeLimit *utls.FakeRecordSizeLimitExtension + DelegatedCredentials *utls.DelegatedCredentialsExtension + SupportedVersions *utls.SupportedVersionsExtension + PSKKeyExchangeModes *utls.PSKKeyExchangeModesExtension + KeyShareCurves *utls.KeyShareExtension +} + +func ToTLSExtensions(e *Extensions) (extensions *TLSExtensions) { + extensions = &TLSExtensions{} + if e == nil { + return extensions + } + if e.SupportedSignatureAlgorithms != nil { + extensions.SupportedSignatureAlgorithms = &utls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []utls.SignatureScheme{}} + for _, s := range e.SupportedSignatureAlgorithms { + extensions.SupportedSignatureAlgorithms.SupportedSignatureAlgorithms = append(extensions.SupportedSignatureAlgorithms.SupportedSignatureAlgorithms, supportedSignatureAlgorithmsExtensions[s]) + } + } + if e.CertCompressionAlgo != nil { + extensions.CertCompressionAlgo = &utls.UtlsCompressCertExtension{Algorithms: []utls.CertCompressionAlgo{}} + for _, s := range e.CertCompressionAlgo { + extensions.CertCompressionAlgo.Algorithms = append(extensions.CertCompressionAlgo.Algorithms, certCompressionAlgoExtensions[s]) + } + } + if e.RecordSizeLimit != 0 { + hexStr := fmt.Sprintf("0x%v", e.RecordSizeLimit) + hexInt, _ := strconv.ParseInt(hexStr, 0, 0) + extensions.RecordSizeLimit = &utls.FakeRecordSizeLimitExtension{uint16(hexInt)} + } + if e.DelegatedCredentials != nil { + extensions.DelegatedCredentials = &utls.DelegatedCredentialsExtension{SupportedSignatureAlgorithms: []utls.SignatureScheme{}} + for _, s := range e.DelegatedCredentials { + extensions.DelegatedCredentials.SupportedSignatureAlgorithms = append(extensions.DelegatedCredentials.SupportedSignatureAlgorithms, supportedSignatureAlgorithmsExtensions[s]) + } + } + if e.SupportedVersions != nil { + extensions.SupportedVersions = &utls.SupportedVersionsExtension{Versions: []uint16{}} + for _, s := range e.SupportedVersions { + extensions.SupportedVersions.Versions = append(extensions.SupportedVersions.Versions, supportedVersionsExtensions[s]) + } + } + if e.PSKKeyExchangeModes != nil { + extensions.PSKKeyExchangeModes = &utls.PSKKeyExchangeModesExtension{Modes: []uint8{}} + for _, s := range e.PSKKeyExchangeModes { + extensions.PSKKeyExchangeModes.Modes = append(extensions.PSKKeyExchangeModes.Modes, pskKeyExchangeModesExtensions[s]) + } + } + if e.KeyShareCurves != nil { + extensions.KeyShareCurves = &utls.KeyShareExtension{KeyShares: []utls.KeyShare{}} + for _, s := range e.KeyShareCurves { + extensions.KeyShareCurves.KeyShares = append(extensions.KeyShareCurves.KeyShares, keyShareCurvesExtensions[s]) + } + } + return extensions +} diff --git a/transport/h2settings.go b/transport/h2settings.go new file mode 100644 index 0000000..99d0197 --- /dev/null +++ b/transport/h2settings.go @@ -0,0 +1,110 @@ +package transport + +import ( + "github.com/wangluozhe/fhttp/http2" +) + +var settings = map[string]http2.SettingID{ + "HEADER_TABLE_SIZE": http2.SettingHeaderTableSize, + "ENABLE_PUSH": http2.SettingEnablePush, + "MAX_CONCURRENT_STREAMS": http2.SettingMaxConcurrentStreams, + "INITIAL_WINDOW_SIZE": http2.SettingInitialWindowSize, + "MAX_FRAME_SIZE": http2.SettingMaxFrameSize, + "MAX_HEADER_LIST_SIZE": http2.SettingMaxHeaderListSize, +} + +type H2Settings struct { + //HEADER_TABLE_SIZE + //ENABLE_PUSH + //MAX_CONCURRENT_STREAMS + //INITIAL_WINDOW_SIZE + //MAX_FRAME_SIZE + //MAX_HEADER_LIST_SIZE + Settings map[string]int + //HEADER_TABLE_SIZE + //ENABLE_PUSH + //MAX_CONCURRENT_STREAMS + //INITIAL_WINDOW_SIZE + //MAX_FRAME_SIZE + //MAX_HEADER_LIST_SIZE + SettingsOrder []string + ConnectionFlow int + HeaderPriority map[string]interface{} + PriorityFrames []map[string]interface{} +} + +func ToHTTP2Settings(h2Settings *H2Settings) (http2Settings *http2.HTTP2Settings) { + http2Settings = &http2.HTTP2Settings{ + Settings: nil, + ConnectionFlow: 0, + HeaderPriority: &http2.PriorityParam{}, + PriorityFrames: nil, + } + if h2Settings.Settings != nil { + if h2Settings.SettingsOrder != nil { + for _, orderKey := range h2Settings.SettingsOrder { + val := h2Settings.Settings[orderKey] + if val != 0 || orderKey == "ENABLE_PUSH" { + http2Settings.Settings = append(http2Settings.Settings, http2.Setting{ + ID: settings[orderKey], + Val: uint32(val), + }) + } + } + } else { + for id, val := range h2Settings.Settings { + http2Settings.Settings = append(http2Settings.Settings, http2.Setting{ + ID: settings[id], + Val: uint32(val), + }) + } + } + } + if h2Settings.ConnectionFlow != 0 { + http2Settings.ConnectionFlow = h2Settings.ConnectionFlow + } + if h2Settings.HeaderPriority != nil { + weight := h2Settings.HeaderPriority["weight"] + var priorityParam *http2.PriorityParam + if weight == nil { + priorityParam = &http2.PriorityParam{ + StreamDep: uint32(h2Settings.HeaderPriority["streamDep"].(int)), + Exclusive: h2Settings.HeaderPriority["exclusive"].(bool), + } + } else { + priorityParam = &http2.PriorityParam{ + StreamDep: uint32(h2Settings.HeaderPriority["streamDep"].(int)), + Exclusive: h2Settings.HeaderPriority["exclusive"].(bool), + Weight: uint8(weight.(int) - 1), + } + } + http2Settings.HeaderPriority = priorityParam + } + if h2Settings.PriorityFrames != nil { + for _, frame := range h2Settings.PriorityFrames { + streamID := frame["streamID"].(int) + priorityParamSource := frame["priorityParam"].(map[string]interface{}) + weight := priorityParamSource["weight"] + var priorityParam http2.PriorityParam + if weight == nil { + priorityParam = http2.PriorityParam{ + StreamDep: uint32(priorityParamSource["streamDep"].(int)), + Exclusive: priorityParamSource["exclusive"].(bool), + } + } else { + priorityParam = http2.PriorityParam{ + StreamDep: uint32(priorityParamSource["streamDep"].(int)), + Exclusive: priorityParamSource["exclusive"].(bool), + Weight: uint8(weight.(int) - 1), + } + } + http2Settings.PriorityFrames = append(http2Settings.PriorityFrames, http2.PriorityFrame{ + FrameHeader: http2.FrameHeader{ + StreamID: uint32(streamID), + }, + PriorityParam: priorityParam, + }) + } + } + return http2Settings +} diff --git a/ja3/transport.go b/transport/roundtripper.go similarity index 61% rename from ja3/transport.go rename to transport/roundtripper.go index 45d5e21..c856e81 100644 --- a/ja3/transport.go +++ b/transport/roundtripper.go @@ -1,71 +1,42 @@ -package ja3 +package transport import ( "context" "errors" "fmt" "net" + "strings" "sync" - http "github.com/Danny-Dasilva/fhttp" - http2 "github.com/Danny-Dasilva/fhttp/http2" + utls "github.com/refraction-networking/utls" + http "github.com/wangluozhe/fhttp" + http2 "github.com/wangluozhe/fhttp/http2" "golang.org/x/net/proxy" - - utls "github.com/Danny-Dasilva/utls" ) var errProtocolNegotiated = errors.New("protocol negotiated") -type Browser struct { +type roundTripper struct { + sync.Mutex + // fix typing JA3 string UserAgent string -} - -func NewJA3Transport(browser Browser, proxyURL string, config *utls.Config) (http.RoundTripper, error) { - if proxyURL != "" { - dialer, err := NewConnectDialer(proxyURL, browser.UserAgent) - if err != nil { - return nil, err - } - if dialer != nil { - - return &JA3Transport{ - dialer: dialer, - - TLSClientConfig: config, - JA3: browser.JA3, - UserAgent: browser.UserAgent, - cachedTransports: make(map[string]http.RoundTripper), - cachedConnections: make(map[string]net.Conn), - }, err - } - } - return &JA3Transport{ - dialer: proxy.Direct, - - TLSClientConfig: config, - JA3: browser.JA3, - UserAgent: browser.UserAgent, - cachedTransports: make(map[string]http.RoundTripper), - cachedConnections: make(map[string]net.Conn), - }, nil -} - -type JA3Transport struct { - sync.Mutex - - JA3 string - UserAgent string - TLSClientConfig *utls.Config cachedConnections map[string]net.Conn cachedTransports map[string]http.RoundTripper - dialer proxy.ContextDialer + dialer proxy.ContextDialer + config *utls.Config + tlsExtensions *TLSExtensions + http2Settings *http2.HTTP2Settings + forceHTTP1 bool } -func (rt *JA3Transport) RoundTrip(req *http.Request) (*http.Response, error) { +func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Header.Get("user-agent") == "" { + req.Header.Set("user-agent", rt.UserAgent) + } addr := rt.getDialTLSAddr(req) if _, ok := rt.cachedTransports[addr]; !ok { if err := rt.getTransport(req, addr); err != nil { @@ -75,7 +46,7 @@ func (rt *JA3Transport) RoundTrip(req *http.Request) (*http.Response, error) { return rt.cachedTransports[addr].RoundTrip(req) } -func (rt *JA3Transport) getTransport(req *http.Request, addr string) error { +func (rt *roundTripper) getTransport(req *http.Request, addr string) error { switch strings.ToLower(req.URL.Scheme) { case "http": rt.cachedTransports[addr] = &http.Transport{DialContext: rt.dialer.DialContext, DisableKeepAlives: true} @@ -98,7 +69,7 @@ func (rt *JA3Transport) getTransport(req *http.Request, addr string) error { return nil } -func (rt *JA3Transport) dialTLS(ctx context.Context, network, addr string) (net.Conn, error) { +func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net.Conn, error) { rt.Lock() defer rt.Unlock() @@ -119,12 +90,14 @@ func (rt *JA3Transport) dialTLS(ctx context.Context, network, addr string) (net. } ////////////////// - spec, err := StringToSpec(rt.JA3, rt.UserAgent) + spec, err := StringToSpec(rt.JA3, rt.UserAgent, rt.tlsExtensions, rt.forceHTTP1) if err != nil { return nil, err } - conn := utls.UClient(rawConn, &utls.Config{ServerName: host, InsecureSkipVerify: true}, // MinVersion: tls.VersionTLS10, + rt.config.ServerName = host + conn := utls.UClient(rawConn, rt.config, + // MinVersion: tls.VersionTLS10, // MaxVersion: tls.VersionTLS13, utls.HelloCustom) @@ -152,29 +125,12 @@ func (rt *JA3Transport) dialTLS(ctx context.Context, network, addr string) (net. // of ALPN. switch conn.ConnectionState().NegotiatedProtocol { case http2.NextProtoTLS: - // t2 := http2.Transport{DialTLS: rt.dialTLSHTTP2} parsedUserAgent := parseUserAgent(rt.UserAgent) - t2 := http2.Transport{DialTLS: rt.dialTLSHTTP2, PushHandler: &http2.DefaultPushHandler{}, Navigator: parsedUserAgent, } - // t2.Settings = []http2.Setting{ - // {ID: http2.SettingMaxHeaderListSize, Val: 262144}, - // {ID: http2.SettingMaxConcurrentStreams, Val: 1000}, - // - // } - //// rTableSize: "HEADER_TABLE_SIZE", - //// SettingEnablePush: "ENABLE_PUSH", - //// SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", - //// SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", - //// SettingMaxFrameSize: "MAX_FRAME_SIZE", - //// SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", - // - // t2.InitialWindowSize = 6291456 - // t2.HeaderTableSize = 65536 - // // t2.PushHandler = &http2.DefaultPushHandler{} - // // rt.cachedTransports[addr] = &t2 + t2.HTTP2Settings = rt.http2Settings rt.cachedTransports[addr] = &t2 default: // Assume the remote peer is speaking HTTP 1.x + TLS. @@ -189,14 +145,48 @@ func (rt *JA3Transport) dialTLS(ctx context.Context, network, addr string) (net. return nil, errProtocolNegotiated } -func (rt *JA3Transport) dialTLSHTTP2(network, addr string, _ *utls.Config) (net.Conn, error) { +func (rt *roundTripper) dialTLSHTTP2(network, addr string, _ *utls.Config) (net.Conn, error) { return rt.dialTLS(context.Background(), network, addr) } -func (rt *JA3Transport) getDialTLSAddr(req *http.Request) string { +func (rt *roundTripper) getDialTLSAddr(req *http.Request) string { host, port, err := net.SplitHostPort(req.URL.Host) if err == nil { return net.JoinHostPort(host, port) } return net.JoinHostPort(req.URL.Host, "443") // we can assume port is 443 at this point } + +func newRoundTripper(browser Browser, config *utls.Config, tlsExtensions *TLSExtensions, http2Settings *http2.HTTP2Settings, forceHTTP1 bool, dialer ...proxy.ContextDialer) http.RoundTripper { + if config == nil { + config = &utls.Config{InsecureSkipVerify: true} + } + if len(dialer) > 0 { + + return &roundTripper{ + dialer: dialer[0], + + JA3: browser.JA3, + UserAgent: browser.UserAgent, + cachedTransports: make(map[string]http.RoundTripper), + cachedConnections: make(map[string]net.Conn), + config: config, + tlsExtensions: tlsExtensions, + http2Settings: http2Settings, + forceHTTP1: forceHTTP1, + } + } + + return &roundTripper{ + dialer: proxy.Direct, + + JA3: browser.JA3, + UserAgent: browser.UserAgent, + cachedTransports: make(map[string]http.RoundTripper), + cachedConnections: make(map[string]net.Conn), + config: config, + tlsExtensions: tlsExtensions, + http2Settings: http2Settings, + forceHTTP1: forceHTTP1, + } +} diff --git a/ja3/utils.go b/transport/utils.go similarity index 74% rename from ja3/utils.go rename to transport/utils.go index 6c13f38..0f69537 100644 --- a/ja3/utils.go +++ b/transport/utils.go @@ -1,11 +1,11 @@ -package ja3 +package transport import ( "crypto/sha256" - "fmt" - utls "github.com/Danny-Dasilva/utls" "strconv" "strings" + + utls "github.com/refraction-networking/utls" ) const ( @@ -25,23 +25,10 @@ func parseUserAgent(userAgent string) string { } -type errExtensionNotExist struct { - Context string -} - -func raiseExtensionError(info string) *errExtensionNotExist { - return &errExtensionNotExist{ - Context: info, - } -} - -func (w *errExtensionNotExist) Error() string { - return fmt.Sprintf("Extension {{ %s }} is not Supported by requests please raise an issue", w.Context) -} - // StringToSpec creates a ClientHelloSpec based on a JA3 string -func StringToSpec(ja3 string, userAgent string) (*utls.ClientHelloSpec, error) { +func StringToSpec(ja3 string, userAgent string, tlsExtensions *TLSExtensions, forceHTTP1 bool) (*utls.ClientHelloSpec, error) { parsedUserAgent := parseUserAgent(userAgent) + ext := tlsExtensions extMap := genMap() tokens := strings.Split(ja3, ",") @@ -58,16 +45,15 @@ func StringToSpec(ja3 string, userAgent string) (*utls.ClientHelloSpec, error) { } // parse curves var targetCurves []utls.CurveID - targetCurves = append(targetCurves, utls.CurveID(utls.GREASE_PLACEHOLDER)) //append grease for Chrome browsers + if parsedUserAgent == chrome { + targetCurves = append(targetCurves, utls.CurveID(utls.GREASE_PLACEHOLDER)) //append grease for Chrome browsers + } for _, c := range curves { cid, err := strconv.ParseUint(c, 10, 16) if err != nil { return nil, err } targetCurves = append(targetCurves, utls.CurveID(cid)) - // if cid != uint64(utls.CurveP521) { - // CurveP521 sometimes causes handshake errors - // } } extMap["10"] = &utls.SupportedCurvesExtension{Curves: targetCurves} @@ -82,17 +68,44 @@ func StringToSpec(ja3 string, userAgent string) (*utls.ClientHelloSpec, error) { } extMap["11"] = &utls.SupportedPointsExtension{SupportedPoints: targetPointFormats} + // force http1 + if forceHTTP1 { + extMap["16"] = &utls.ALPNExtension{ + AlpnProtocols: []string{"http/1.1"}, + } + } + + // custom tls extensions + if tlsExtensions != nil { + if ext.SupportedSignatureAlgorithms != nil { + extMap["13"] = ext.SupportedSignatureAlgorithms + } + if ext.CertCompressionAlgo != nil { + extMap["27"] = ext.CertCompressionAlgo + } + if ext.RecordSizeLimit != nil { + extMap["28"] = ext.RecordSizeLimit + } + if ext.DelegatedCredentials != nil { + extMap["34"] = ext.DelegatedCredentials + } + if ext.SupportedVersions != nil { + extMap["43"] = ext.SupportedVersions + } + if ext.PSKKeyExchangeModes != nil { + extMap["45"] = ext.PSKKeyExchangeModes + } + if ext.KeyShareCurves != nil { + extMap["51"] = ext.KeyShareCurves + } + } + // set extension 43 vid64, err := strconv.ParseUint(version, 10, 16) if err != nil { return nil, err } vid := uint16(vid64) - // extMap["43"] = &utls.SupportedVersionsExtension{ - // Versions: []uint16{ - // utls.VersionTLS12, - // }, - // } // build extenions list var exts []utls.TLSExtension @@ -111,16 +124,6 @@ func StringToSpec(ja3 string, userAgent string) (*utls.ClientHelloSpec, error) { } exts = append(exts, te) } - //Add this back in if user agent is chrome and no padding extension is given - // if parsedUserAgent == chrome { - // exts = append(exts, &utls.UtlsGREASEExtension{}) - // exts = append(exts, &utls.UtlsPaddingExtension{GetPaddingLen: utls.BoringPaddingStyle}) - // } - // build SSLVersion - // vid64, err := strconv.ParseUint(version, 10, 16) - // if err != nil { - // return nil, err - // } // build CipherSuites var suites []uint16 @@ -176,19 +179,29 @@ func genMap() (extMap map[string]utls.TLSExtension) { "21": &utls.UtlsPaddingExtension{GetPaddingLen: utls.BoringPaddingStyle}, "22": &utls.GenericExtension{Id: 22}, // encrypt_then_mac "23": &utls.UtlsExtendedMasterSecretExtension{}, - "27": &utls.CompressCertificateExtension{ + "27": &utls.UtlsCompressCertExtension{ Algorithms: []utls.CertCompressionAlgo{utls.CertCompressionBrotli}, }, - "28": &utls.FakeRecordSizeLimitExtension{}, //Limit: 0x4001 + "28": &utls.FakeRecordSizeLimitExtension{ + Limit: 0x4001, + }, //Limit: 0x4001 + "34": &utls.DelegatedCredentialsExtension{ + SupportedSignatureAlgorithms: []utls.SignatureScheme{ + utls.ECDSAWithP256AndSHA256, + utls.ECDSAWithP384AndSHA384, + utls.ECDSAWithP521AndSHA512, + utls.ECDSAWithSHA1, + }, + }, "35": &utls.SessionTicketExtension{}, - "34": &utls.GenericExtension{Id: 34}, - "41": &utls.GenericExtension{Id: 41}, //FIXME pre_shared_key + //"41": &utls.GenericExtension{Id: 41}, //FIXME pre_shared_key, Currently not supported 41 extension "43": &utls.SupportedVersionsExtension{Versions: []uint16{ utls.GREASE_PLACEHOLDER, utls.VersionTLS13, utls.VersionTLS12, utls.VersionTLS11, - utls.VersionTLS10}}, + utls.VersionTLS10, + }}, "44": &utls.CookieExtension{}, "45": &utls.PSKKeyExchangeModesExtension{Modes: []uint8{ utls.PskModeDHE, @@ -201,13 +214,13 @@ func genMap() (extMap map[string]utls.TLSExtension) { // {Group: utls.CurveP384}, known bug missing correct extensions for handshake }}, - "30032": &utls.GenericExtension{Id: 0x7550, Data: []byte{0}}, //FIXME "13172": &utls.NPNExtension{}, "17513": &utls.ApplicationSettingsExtension{ - SupportedALPNList: []string{ + SupportedProtocols: []string{ "h2", }, }, + "30032": &utls.GenericExtension{Id: 0x7550, Data: []byte{0}}, //FIXME "65281": &utls.RenegotiationInfoExtension{ Renegotiation: utls.RenegotiateOnceAsClient, }, diff --git a/url/Cookies.go b/url/Cookies.go index 87ad442..9cb2c2c 100644 --- a/url/Cookies.go +++ b/url/Cookies.go @@ -2,8 +2,8 @@ package url import ( "errors" - http "github.com/Danny-Dasilva/fhttp" - "github.com/Danny-Dasilva/fhttp/cookiejar" + http "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/fhttp/cookiejar" "net/url" "strings" ) diff --git a/url/Headers.go b/url/Headers.go index 9e0eeb5..4d317a5 100644 --- a/url/Headers.go +++ b/url/Headers.go @@ -2,7 +2,7 @@ package url import ( "errors" - http "github.com/Danny-Dasilva/fhttp" + http "github.com/wangluozhe/fhttp" "strings" ) diff --git a/url/Request.go b/url/Request.go index 4948867..5af0c15 100644 --- a/url/Request.go +++ b/url/Request.go @@ -1,8 +1,10 @@ package url import ( - "github.com/Danny-Dasilva/fhttp" - "github.com/Danny-Dasilva/fhttp/cookiejar" + "github.com/wangluozhe/fhttp" + "github.com/wangluozhe/fhttp/cookiejar" + "github.com/wangluozhe/fhttp/http2" + ja3 "github.com/wangluozhe/requests/transport" "time" ) @@ -27,4 +29,6 @@ type Request struct { Verify bool Cert []string Ja3 string + TLSExtensions *ja3.TLSExtensions + HTTP2Settings *http2.HTTP2Settings } diff --git a/version.go b/version.go index 7cf1c72..88b3ba9 100644 --- a/version.go +++ b/version.go @@ -1,7 +1,7 @@ package requests const ( - NAME = "golang-requests" // 名称 - VERSION = "1.0.42" // 当前版本 - USER_AGENT = NAME + VERSION // UA + NAME = "golang-requests" // 名称 + VERSION = "1.1.0" // 当前版本 + USER_AGENT = NAME + VERSION // UA )