Releases/v7.21.0 (#128)

.github/workflows/ci-test.yml

@@ -25,6 +25,8 @@ jobs:
           GOPATH=$GITHUB_WORKSPACE go get golang.org/x/sync/singleflight
           GOPATH=$GITHUB_WORKSPACE go get github.com/qiniu/dyn
           GOPATH=$GITHUB_WORKSPACE go get github.com/gofrs/flock
+          GOPATH=$GITHUB_WORKSPACE go get github.com/alex-ant/gomath/rational
+          GOPATH=$GITHUB_WORKSPACE go get github.com/matishsiao/goInfo
 
           # FIXME special package
           # github.com/go-playground/validator/v10
@@ -54,6 +56,7 @@ jobs:
           GOPATH=$GITHUB_WORKSPACE go get github.com/davecgh/go-spew/spew
           GOPATH=$GITHUB_WORKSPACE go get github.com/pmezard/go-difflib/difflib
           GOPATH=$GITHUB_WORKSPACE go get gopkg.in/yaml.v3
+          rm -rf $GITHUB_WORKSPACE/src/github.com/BurntSushi/toml && git clone -b v0.3.1 --depth 1 https://github.com/BurntSushi/toml.git $GITHUB_WORKSPACE/src/github.com/BurntSushi/toml
 
           GOPATH=$GITHUB_WORKSPACE make unittest
         working-directory: src/github.com/qiniu/go-sdk

Makefile

@@ -4,6 +4,9 @@ test:
 unittest:
 	go test -tags=unit -failfast -v -coverprofile=coverage.txt `go list ./... | egrep -v 'examples|sms'`
 
+integrationtest:
+	go test -tags=integration -failfast -parallel 1 -v -coverprofile=coverage.txt `go list ./... | egrep -v 'examples|sms'`
+
 staticcheck:
 	staticcheck -go 1.10 `go list ./... | egrep -v 'examples|sms'`
 

auth/credentials.go

@@ -14,6 +14,7 @@ import (
 	api "github.com/qiniu/go-sdk/v7"
 	"github.com/qiniu/go-sdk/v7/conf"
 	internal_io "github.com/qiniu/go-sdk/v7/internal/io"
+	"github.com/qiniu/go-sdk/v7/storagev2/defaults"
 )
 
 const (
@@ -32,18 +33,39 @@ type Credentials struct {
 	SecretKey []byte
 }
 
-// 构建一个Credentials对象
+// New 构建一个Credentials对象
 func New(accessKey, secretKey string) *Credentials {
 	return &Credentials{accessKey, []byte(secretKey)}
 }
 
+// Default 构建默认的 Credentials 对象
+func Default() *Credentials {
+	accessKey, secretKey, err := defaults.Credentials()
+	if err == nil && accessKey != "" && secretKey != "" {
+		return New(accessKey, secretKey)
+	}
+	return nil
+}
+
 // Sign 对数据进行签名，一般用于私有空间下载用途
 func (ath *Credentials) Sign(data []byte) (token string) {
-	h := hmac.New(sha1.New, ath.SecretKey)
+	var (
+		accessKey string
+		secretKey []byte
+	)
+	if ath == nil {
+		if cred := Default(); cred != nil {
+			accessKey = cred.AccessKey
+			secretKey = cred.SecretKey
+		}
+	} else {
+		accessKey, secretKey = ath.AccessKey, ath.SecretKey
+	}
+	h := hmac.New(sha1.New, secretKey)
 	h.Write(data)
 
 	sign := base64.URLEncoding.EncodeToString(h.Sum(nil))
-	return fmt.Sprintf("%s:%s", ath.AccessKey, sign)
+	return fmt.Sprintf("%s:%s", accessKey, sign)
 }
 
 // SignToken 根据t的类型对请求进行签名，并把token加入req中
@@ -74,8 +96,16 @@ func (ath *Credentials) SignWithData(b []byte) (token string) {
 
 // IsIAMKey 判断AccessKey是否为IAM的Key
 func (ath *Credentials) IsIAMKey() bool {
-	return len(ath.AccessKey) == IAMKeyLen*4/3 &&
-		strings.HasPrefix(ath.AccessKey, IAMKeyPrefix)
+	var accessKey string
+	if ath == nil {
+		if cred := Default(); cred != nil {
+			accessKey = cred.AccessKey
+		}
+	} else {
+		accessKey = ath.AccessKey
+	}
+	return len(accessKey) == IAMKeyLen*4/3 &&
+		strings.HasPrefix(accessKey, IAMKeyPrefix)
 }
 
 func collectData(req *http.Request) (data []byte, err error) {

cdn/api.go

@@ -24,6 +24,9 @@ type CdnManager struct {
 
 // NewCdnManager 用来构建一个新的 CdnManager
 func NewCdnManager(mac *auth.Credentials) *CdnManager {
+	if mac == nil {
+		mac = auth.Default()
+	}
 	return &CdnManager{mac: mac}
 }
 

client/client.go

@@ -21,16 +21,14 @@ import (
 	"github.com/qiniu/go-sdk/v7/reqid"
 )
 
-var UserAgent = getUserAgentWithAppName("default")
-var DefaultClient = Client{
-	&http.Client{
-		Transport: http.DefaultTransport,
-	},
-}
+var (
+	UserAgent     = getUserAgentWithAppName("default")
+	DefaultClient = Client{&http.Client{Transport: DefaultTransport}}
 
-// 用来打印调试信息
-var DebugMode = false
-var DeepDebugInfo = false
+	// 用来打印调试信息
+	DebugMode     = false
+	DeepDebugInfo = false
+)
 
 // --------------------------------------------------------------------
 
@@ -51,8 +49,12 @@ func SetAppName(userApp string) error {
 }
 
 func getUserAgentWithAppName(userApp string) string {
-	return fmt.Sprintf("QiniuGo/%s (%s; %s; %s) %s",
-		conf.Version, runtime.GOOS, runtime.GOARCH, userApp, runtime.Version())
+	userAgentPrefix := "QiniuGo/"
+	if testRuntime {
+		userAgentPrefix = "QiniuGo_Debug/"
+	}
+	return fmt.Sprintf("%s%s (%s; %s; %s) %s",
+		userAgentPrefix, conf.Version, runtime.GOOS, runtime.GOARCH, userApp, runtime.Version())
 }
 
 // --------------------------------------------------------------------

client/client_1.12.go

@@ -0,0 +1,18 @@
+//go:build !1.13
+// +build !1.13
+
+package client
+
+import (
+	"net/http"
+	"time"
+)
+
+var DefaultTransport http.RoundTripper = &http.Transport{
+	Proxy:                 http.ProxyFromEnvironment,
+	DialContext:           defaultDialFunc,
+	MaxIdleConns:          100,
+	IdleConnTimeout:       90 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+}

client/client_1.13.go

@@ -0,0 +1,19 @@
+//go:build 1.13
+// +build 1.13
+
+package client
+
+import (
+	"net/http"
+	"time"
+)
+
+var DefaultTransport http.RoundTripper = &http.Transport{
+	Proxy:                 http.ProxyFromEnvironment,
+	DialContext:           defaultDialFunc,
+	ForceAttemptHTTP2:     true,
+	MaxIdleConns:          100,
+	IdleConnTimeout:       90 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+}

client/client_prod_env.go

@@ -0,0 +1,6 @@
+//go:build !unit && !integration
+// +build !unit,!integration
+
+package client
+
+const testRuntime = false

client/client_test_env.go

@@ -0,0 +1,6 @@
+//go:build unit || integration
+// +build unit integration
+
+package client
+
+const testRuntime = true

client/dialer.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"net"
+	"time"
+)
+
+type (
+	resolverContextKey          struct{}
+	dialTimeoutContextKey       struct{}
+	keepAliveIntervalContextKey struct{}
+	resolverContextValue        struct {
+		domain string
+		ips    []net.IP
+	}
+)
+
+func defaultDialFunc(ctx context.Context, network string, address string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(address)
+	if err != nil {
+		host = address
+	}
+
+	dialTimeout, ok := ctx.Value(dialTimeoutContextKey{}).(time.Duration)
+	if !ok {
+		dialTimeout = 30 * time.Second
+	}
+	keepAliveInterval, ok := ctx.Value(keepAliveIntervalContextKey{}).(time.Duration)
+	if !ok {
+		keepAliveInterval = 15 * time.Second
+	}
+	if resolved, ok := ctx.Value(resolverContextKey{}).(resolverContextValue); ok && len(resolved.ips) > 0 && resolved.domain == host {
+		dialer := net.Dialer{Timeout: dialTimeout / time.Duration(len(resolved.ips)), KeepAlive: keepAliveInterval}
+		for _, ip := range resolved.ips {
+			newAddr := ip.String()
+			if port != "" {
+				newAddr = net.JoinHostPort(newAddr, port)
+			}
+			if conn, err := dialer.DialContext(ctx, network, newAddr); err == nil {
+				return conn, nil
+			}
+		}
+	}
+	return (&net.Dialer{Timeout: dialTimeout, KeepAlive: keepAliveInterval}).DialContext(ctx, network, address)
+}
+
+func WithResolvedIPs(ctx context.Context, domain string, ips []net.IP) context.Context {
+	return context.WithValue(ctx, resolverContextKey{}, resolverContextValue{domain: domain, ips: ips})
+}
+
+func WithDialTimeout(ctx context.Context, timeout time.Duration) context.Context {
+	return context.WithValue(ctx, dialTimeoutContextKey{}, timeout)
+}
+
+func WithKeepAliveInterval(ctx context.Context, interval time.Duration) context.Context {
+	return context.WithValue(ctx, keepAliveIntervalContextKey{}, interval)
+}

client/dialer_test.go

@@ -0,0 +1,37 @@
+//go:build unit
+// +build unit
+
+package client
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestDefaultDialer(t *testing.T) {
+	var responseBody struct {
+		Status string `json:"status"`
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("Content-Type", "application/json")
+		w.Write([]byte(`{"status":"ok"}`))
+	}))
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	port := server.Listener.Addr().(*net.TCPAddr).Port
+
+	ctx := WithResolvedIPs(context.Background(), "www.qiniu.com", []net.IP{net.IPv4(127, 0, 0, 1)})
+	err := DefaultClient.Call(ctx, &responseBody, http.MethodGet, fmt.Sprintf("http://www.qiniu.com:%d/", port), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if responseBody.Status != "ok" {
+		t.Fatal("unexpected response")
+	}
+}

conf/conf.go

@@ -1,8 +1,7 @@
 package conf
 
 import (
-	"os"
-	"strings"
+	"github.com/qiniu/go-sdk/v7/internal/env"
 )
 
 const Version = "7.20.2"
@@ -12,12 +11,9 @@ const (
 	CONTENT_TYPE_FORM      = "application/x-www-form-urlencoded"
 	CONTENT_TYPE_OCTET     = "application/octet-stream"
 	CONTENT_TYPE_MULTIPART = "multipart/form-data"
-
-	disableQiniuTimestampSignatureEnvKey = "DISABLE_QINIU_TIMESTAMP_SIGNATURE"
 )
 
 func IsDisableQiniuTimestampSignature() bool {
-	value := os.Getenv(disableQiniuTimestampSignatureEnvKey)
-	value = strings.ToLower(value)
-	return value == "true" || value == "yes" || value == "y" || value == "1"
+	isDisabled, _ := env.DisableQiniuTimestampSignatureFromEnvironment()
+	return isDisabled
 }

examples/form_upload_simple.go

@@ -3,12 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
-	"net"
 	"os"
 
-	"net/http"
-	"net/url"
-
 	"github.com/qiniu/go-sdk/v7/auth"
 	"github.com/qiniu/go-sdk/v7/storage"
 )
@@ -17,11 +13,11 @@ var (
 	accessKey = os.Getenv("QINIU_ACCESS_KEY")
 	secretKey = os.Getenv("QINIU_SECRET_KEY")
 	bucket    = os.Getenv("QINIU_TEST_BUCKET")
+	localFile = os.Getenv("LOCAL_FILE")
+	key       = os.Getenv("KEY")
 )
 
 func main() {
-	localFile := "/Users/jemy/Documents/github.png"
-	key := "github-x.png"
 	putPolicy := storage.PutPolicy{
 		Scope: bucket + ":" + key,
 	}
@@ -36,36 +32,15 @@ func main() {
 	// 上传是否使用CDN上传加速
 	cfg.UseCdnDomains = false
 
-	//设置代理
-	proxyURL := "http://localhost:8888"
-	proxyURI, _ := url.Parse(proxyURL)
-
-	//绑定网卡
-	nicIP := "100.100.33.138"
-	dialer := &net.Dialer{
-		LocalAddr: &net.TCPAddr{
-			IP: net.ParseIP(nicIP),
-		},
-	}
-
-	//构建代理client对象
-	client := http.Client{
-		Transport: &http.Transport{
-			Proxy: http.ProxyURL(proxyURI),
-			Dial:  dialer.Dial,
-		},
-	}
-
 	// 构建表单上传的对象
-	formUploader := storage.NewFormUploaderEx(&cfg, &storage.Client{Client: &client})
+	formUploader := storage.NewFormUploader(&cfg)
 	ret := storage.PutRet{}
 	// 可选配置
 	putExtra := storage.PutExtra{
 		Params: map[string]string{
 			"x:name": "github logo",
 		},
 	}
-	//putExtra.NoCrc32Check = true
 	err := formUploader.PutFile(context.Background(), &ret, upToken, key, localFile, &putExtra)
 	if err != nil {
 		fmt.Println(err)