check bucketname, dns scatter

auth.go

@@ -1,14 +1,18 @@
 package cos
 
 import (
+	"context"
 	"crypto/hmac"
 	"crypto/sha1"
 	"encoding/json"
 	"fmt"
 	"hash"
 	"io/ioutil"
+	math_rand "math/rand"
+	"net"
 	"net/http"
 	"net/url"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
@@ -26,8 +30,52 @@ var (
 	defaultCVMSchema     = "http"
 	defaultCVMMetaHost   = "metadata.tencentyun.com"
 	defaultCVMCredURI    = "latest/meta-data/cam/security-credentials"
+	internalHost         = regexp.MustCompile(`^.*cos-internal\.[a-z-1]+\.tencentcos\.cn$`)
 )
 
+var DNSScatterDialContext = DNSScatterDialContextFunc
+
+var DNSScatterTransport = &http.Transport{
+	Proxy:                 http.ProxyFromEnvironment,
+	DialContext:           DNSScatterDialContext,
+	MaxIdleConns:          100,
+	IdleConnTimeout:       90 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+}
+
+func DNSScatterDialContextFunc(ctx context.Context, network string, addr string) (conn net.Conn, err error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	dialer := net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+		DualStack: true,
+	}
+	// DNS 打散
+	math_rand.Seed(time.Now().UnixNano())
+	start := math_rand.Intn(len(ips))
+	for i := start; i < len(ips); i++ {
+		conn, err = dialer.DialContext(ctx, network, net.JoinHostPort(ips[i].IP.String(), port))
+		if err == nil {
+			return
+		}
+	}
+	for i := 0; i < start; i++ {
+		conn, err = dialer.DialContext(ctx, network, net.JoinHostPort(ips[i].IP.String(), port))
+		if err == nil {
+			break
+		}
+	}
+	return
+}
+
 // 需要校验的 Headers 列表
 var NeedSignHeaders = map[string]bool{
 	"host":                           true,
@@ -319,14 +367,18 @@ func (t *AuthorizationTransport) RoundTrip(req *http.Request) (*http.Response, e
 	authTime := NewAuthTime(defaultAuthExpire)
 	AddAuthorizationHeader(ak, sk, token, req, authTime)
 
-	resp, err := t.transport().RoundTrip(req)
+	resp, err := t.transport(req).RoundTrip(req)
 	return resp, err
 }
 
-func (t *AuthorizationTransport) transport() http.RoundTripper {
+func (t *AuthorizationTransport) transport(req *http.Request) http.RoundTripper {
 	if t.Transport != nil {
 		return t.Transport
 	}
+	// 内部域名默认使用DNS打散
+	if rc := internalHost.MatchString(req.URL.Hostname()); rc {
+		return DNSScatterTransport
+	}
 	return http.DefaultTransport
 }
 

cos.go

@@ -15,6 +15,7 @@ import (
 	"text/template"
 	"time"
 
+	"regexp"
 	"strconv"
 
 	"github.com/google/go-querystring/query"
@@ -29,10 +30,17 @@ const (
 	defaultServiceBaseURL = "http://service.cos.myqcloud.com"
 )
 
-var bucketURLTemplate = template.Must(
-	template.New("bucketURLFormat").Parse(
-		"{{.Schema}}://{{.BucketName}}.cos.{{.Region}}.myqcloud.com",
-	),
+var (
+	bucketURLTemplate = template.Must(
+		template.New("bucketURLFormat").Parse(
+			"{{.Schema}}://{{.BucketName}}.cos.{{.Region}}.myqcloud.com",
+		),
+	)
+
+	// {<http://>|<https://>}{bucketname-appid}.{cos|cos-internal|cos-website|ci}.{region}.{myqcloud.com/tencentcos.cn}{/}
+	hostSuffix       = regexp.MustCompile(`^.*((cos|cos-internal|cos-website|ci)\.[a-z-1]+|file)\.(myqcloud\.com|tencentcos\.cn).*$`)
+	hostPrefix       = regexp.MustCompile(`^(http://|https://){0,1}([a-z0-9-]+-[0-9]+\.){0,1}((cos|cos-internal|cos-website|ci)\.[a-z-1]+|file)\.(myqcloud\.com|tencentcos\.cn).*$`)
+	invalidBucketErr = fmt.Errorf("invalid bucket format, please check your cos.BaseURL")
 )
 
 // BaseURL 访问各 API 所需的基础 URL
@@ -192,6 +200,9 @@ func (c *Client) GetCredential() *Credential {
 }
 
 func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) {
+	if !checkURL(baseURL) {
+		return nil, invalidBucketErr
+	}
 	uri, err = addURLOptions(uri, optQuery)
 	if err != nil {
 		return
@@ -352,7 +363,7 @@ func (c *Client) doRetry(ctx context.Context, opt *sendOptions) (resp *Response,
 	interval := c.Conf.RetryOpt.Interval
 	for nr < count {
 		resp, err = c.send(ctx, opt)
-		if err != nil {
+		if err != nil && err != invalidBucketErr {
 			if resp != nil && resp.StatusCode <= 499 {
 				dobreak := true
 				for _, v := range c.Conf.RetryOpt.StatusCode {
@@ -439,6 +450,14 @@ func addHeaderOptions(header http.Header, opt interface{}) (http.Header, error)
 	return header, nil
 }
 
+func checkURL(baseURL *url.URL) bool {
+	host := baseURL.String()
+	if hostSuffix.MatchString(host) && !hostPrefix.MatchString(host) {
+		return false
+	}
+	return true
+}
+
 // Owner defines Bucket/Object's owner
 type Owner struct {
 	UIN         string `xml:"uin,omitempty"`

debug/http.go

@@ -2,10 +2,12 @@ package debug
 
 import (
 	"fmt"
+	"github.com/tencentyun/cos-go-sdk-v5"
 	"io"
 	"net/http"
 	"net/http/httputil"
 	"os"
+	"regexp"
 )
 
 // DebugRequestTransport 会打印请求和响应信息, 方便调试.
@@ -21,6 +23,8 @@ type DebugRequestTransport struct {
 	Transport http.RoundTripper
 }
 
+var internalHost = regexp.MustCompile(`^.*cos-internal\.[a-z-1]+\.tencentcos\.cn$`)
+
 // RoundTrip implements the RoundTripper interface.
 func (t *DebugRequestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	req = cloneRequest(req) // per RoundTrip contract
@@ -34,7 +38,7 @@ func (t *DebugRequestTransport) RoundTrip(req *http.Request) (*http.Response, er
 		fmt.Fprintf(w, "%s\n\n", string(a))
 	}
 
-	resp, err := t.transport().RoundTrip(req)
+	resp, err := t.transport(req).RoundTrip(req)
 	if err != nil {
 		return resp, err
 	}
@@ -48,10 +52,14 @@ func (t *DebugRequestTransport) RoundTrip(req *http.Request) (*http.Response, er
 	return resp, err
 }
 
-func (t *DebugRequestTransport) transport() http.RoundTripper {
+func (t *DebugRequestTransport) transport(req *http.Request) http.RoundTripper {
 	if t.Transport != nil {
 		return t.Transport
 	}
+	// 内部域名默认使用DNS打散
+	if rc := internalHost.MatchString(req.URL.Hostname()); rc {
+		return cos.DNSScatterTransport
+	}
 	return http.DefaultTransport
 }
 

example/object/get_with_cos_internal.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"github.com/tencentyun/cos-go-sdk-v5"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+func log_status(err error) {
+	if err == nil {
+		return
+	}
+	if cos.IsNotFoundError(err) {
+		// WARN
+		fmt.Println("WARN: Resource is not existed")
+	} else if e, ok := cos.IsCOSError(err); ok {
+		fmt.Printf("ERROR: Code: %v\n", e.Code)
+		fmt.Printf("ERROR: Message: %v\n", e.Message)
+		fmt.Printf("ERROR: Resource: %v\n", e.Resource)
+		fmt.Printf("ERROR: RequestId: %v\n", e.RequestID)
+		// ERROR
+	} else {
+		fmt.Printf("ERROR: %v\n", err)
+		// ERROR
+	}
+}
+
+func main() {
+	// 存储桶名称，由bucketname-appid 组成，appid必须填入，可以在COS控制台查看存储桶名称。 https://console.cloud.tencent.com/cos5/bucket
+	// 替换为用户的 region，存储桶region可以在COS控制台“存储桶概览”查看 https://console.cloud.tencent.com/ ，关于地域的详情见 https://cloud.tencent.com/document/product/436/6224 。
+	u, _ := url.Parse("http://test-1259654469.cos-internal.ap-guangzhou.tencentcos.cn")
+	b := &cos.BaseURL{BucketURL: u}
+	c := cos.NewClient(b, &http.Client{
+		Transport: &cos.AuthorizationTransport{
+			// 通过环境变量获取密钥
+			// 环境变量 COS_SECRETID 表示用户的 SecretId，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
+			SecretID: os.Getenv("COS_SECRETID"),
+			// 环境变量 COS_SECRETKEY 表示用户的 SecretKey，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
+			SecretKey: os.Getenv("COS_SECRETKEY"),
+			// 1、不设置 Transport 时：
+			//		1）对于内部域名，Transport 默认使用 &cos.DNSScatterTransport, cos.DNSScatterTransport 获取DNS响应后，随机获取IP
+			//		2）对于其他域名，Transport 默认使用 &http.DefaultTransport (go http 库默认 Transport)
+			// 2、设置 Transport 时，以用户设置为准
+			// Transport: http.DefaultTransport,
+		},
+	})
+
+	name := "exampleobject"
+	_, err := c.Object.Get(context.Background(), name, nil)
+	log_status(err)
+}