diff --git a/openstack/obs/CHANGELOG b/openstack/obs/CHANGELOG new file mode 100644 index 000000000..5656d7b13 --- /dev/null +++ b/openstack/obs/CHANGELOG @@ -0,0 +1,51 @@ +Version 3.19.11 +Updated the version ID format. The new version ID is named in the following format: Main version ID.Year ID.Month ID. + +New Features: + +Documentation & Demo: +1. Added the description of WithMaxRedirectCount and WithSecurityToken in section "ObsClient Initialization" in the API Reference. +2. Added descriptions about WithMaxRedirectCount and WithSecurityToken in section "Configuring an Instance of OBSClient" in the Developer Guide. +3. Added methods for handling HTTP status code 405 in section "FAQs" in the Developer Guide. + +Resolved Issues: +1. Fixed the issue that error code 400 is returned when DisplayName is carried in some circumstances. +2. Fixed the issue that error code 400 is returned when the SetBucketNotification API is called. +3. Fixed the issue that the SetObjectAcl API does not support the specified Delivered parameter. +4. Fixed the issue that the SetBucketLoggingConfiguration API is incompatible with the Agency field in different protocols. +5. Fixed the issue that the SetBucketLoggingConfiguration API incorrectly processes the Delivered field. +6. Fixed the issue that the ContentMD5 configuration does not take effect when the UploadPart API is called. +7. Fixed the issue that the SetBucketStoragePolicy API processes incorrect storage classes inconsistently in different protocols. +8. Fixed the issue that error code 400 is returned because the CreateBucket API does not distinguish protocols. +9. Fixed the syntax issue of the getBucketAclObs API. +10. Rectified the error of the SetBucketWebsiteConfiguration API. +11. Fixed the compatibility issue of the temporary authentication API in different protocols. +12. Fixed the issue that the authentication information header is added when redirection is performed upon a 302 response returned for a GET request. +13. Fixed the issue that the content-type cannot be obtained based on the file name extension if the extension is in uppercase. +14. Changed the content-type of files with the .svg extension to image/svg+xml. +15. Fixed the issue that an incorrect API is called in the sample code for deleting a bucket in examples/bucket_operations_sample.go. +16. Fixed the issue in calculating the authentication value during the access using temporary access keys. +17. Fixed the issue that some response fields are empty in anonymous access. + +----------------------------------------------------------------------------------- + +Version 3.1.2 + +New Features: + +Documentation & Demo: + +Resolved Issues: +1. Fixed the issue that the configuration of ContentType does not take effect when Obs.InitiateMultipartUpload is called. + +----------------------------------------------------------------------------------- + +Version 3.1.1 + +New Features: +1. Supports access to OBS using a user-defined domain name (obs.WithCustomDomainName). +2. The API for object upload (ObsClient.PutObject) can automatically identify a wider MIME type. + +Resolved Issues: +1. Fixed the issue that a null pointer error is reported when ObsClient.GetBucketStoragePolicy is called. +2. Fixed the 400 error reported by ObsClient.SetBucketStoragePolicy. diff --git a/openstack/obs/README.MD b/openstack/obs/README.MD new file mode 100644 index 000000000..849dc40a3 --- /dev/null +++ b/openstack/obs/README.MD @@ -0,0 +1,4 @@ +Version:3.19.11 + +You can get the latest source code from [here](https://obssdk.obs.cn-north-1.myhuaweicloud.com/current/go/go.zip), +then unzip the file. diff --git a/openstack/obs/auth.go b/openstack/obs/auth.go index c1d980306..17ccde206 100644 --- a/openstack/obs/auth.go +++ b/openstack/obs/auth.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -10,25 +22,47 @@ import ( func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params map[string]string, headers map[string][]string, expires int64) (requestUrl string, err error) { - - isV4 := obsClient.conf.signature == SignatureV4 - - requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params) + isAkSkEmpty := obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" + if isAkSkEmpty == false && obsClient.conf.securityProvider.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = obsClient.conf.securityProvider.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + } + } + requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true) parsedRequestUrl, err := url.Parse(requestUrl) if err != nil { return "", err } encodeHeaders(headers) - hostName := parsedRequestUrl.Host - skipAuth := obsClient.prepareHeaders(headers, hostName, isV4) + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) - if !skipAuth { + if isAkSkEmpty { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { if isV4 { - date, _ := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + date, parseDateErr := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } + delete(headers, HEADER_DATE_CAMEL) shortDate := date.Format(SHORT_DATE_FORMAT) longDate := date.Format(LONG_DATE_FORMAT) + if len(headers[HEADER_HOST_CAMEL]) != 0 { + index := strings.LastIndex(headers[HEADER_HOST_CAMEL][0], ":") + if index != -1 { + port := headers[HEADER_HOST_CAMEL][0][index+1:] + if port == "80" || port == "443" { + headers[HEADER_HOST_CAMEL] = []string{headers[HEADER_HOST_CAMEL][0][:index]} + } + } + + } signedHeaders, _headers := getSignedHeaders(headers) @@ -39,8 +73,13 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params[PARAM_EXPIRES_AMZ_CAMEL] = Int64ToString(expires) params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = strings.Join(signedHeaders, ";") - requestUrl, canonicalizedUrl = obsClient.conf.formatUrls(bucketName, objectKey, params) - parsedRequestUrl, _ = url.Parse(requestUrl) + requestUrl, canonicalizedUrl = obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestUrl, _err := url.Parse(requestUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl with reason: %v", _err) + return "", _err + } + stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, signedHeaders, _headers) signature := getSignature(stringToSign, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) @@ -48,26 +87,82 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, } else { originDate := headers[HEADER_DATE_CAMEL][0] - date, _ := time.Parse(RFC1123_FORMAT, originDate) + date, parseDateErr := time.Parse(RFC1123_FORMAT, originDate) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } expires += date.Unix() headers[HEADER_DATE_CAMEL] = []string{Int64ToString(expires)} - stringToSign := getV2StringToSign(method, canonicalizedUrl, headers) + stringToSign := getV2StringToSign(method, canonicalizedUrl, headers, obsClient.conf.signature == SignatureObs) signature := UrlEncode(Base64Encode(HmacSha1([]byte(obsClient.conf.securityProvider.sk), []byte(stringToSign))), false) if strings.Index(requestUrl, "?") < 0 { requestUrl += "?" } else { requestUrl += "&" } - headers[HEADER_DATE_CAMEL] = []string{originDate} - requestUrl += fmt.Sprintf("AWSAccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(obsClient.conf.securityProvider.ak, false), - expires, signature) + delete(headers, HEADER_DATE_CAMEL) + + if obsClient.conf.signature != SignatureObs { + requestUrl += "AWS" + } + requestUrl += fmt.Sprintf("AccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(obsClient.conf.securityProvider.ak, false), expires, signature) + } + } + + return +} + +func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, hostName string) (requestUrl string, err error) { + isAkSkEmpty := obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" + if isAkSkEmpty == false && obsClient.conf.securityProvider.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + headers[HEADER_STS_TOKEN_OBS] = []string{obsClient.conf.securityProvider.securityToken} + } else { + headers[HEADER_STS_TOKEN_AMZ] = []string{obsClient.conf.securityProvider.securityToken} + } + } + isObs := obsClient.conf.signature == SignatureObs + requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestUrl, err := url.Parse(requestUrl) + if err != nil { + return "", err + } + encodeHeaders(headers) + + if hostName == "" { + hostName = parsedRequestUrl.Host + } + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if isAkSkEmpty { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + ak := obsClient.conf.securityProvider.ak + sk := obsClient.conf.securityProvider.sk + var authorization string + if isV4 { + headers[HEADER_CONTENT_SHA256_AMZ] = []string{EMPTY_CONTENT_SHA256} + ret := v4Auth(ak, sk, obsClient.conf.region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, headers) + authorization = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else { + ret := v2Auth(ak, sk, method, canonicalizedUrl, headers, isObs) + hashPrefix := V2_HASH_PREFIX + if isObs { + hashPrefix = OBS_HASH_PREFIX + } + authorization = fmt.Sprintf("%s %s:%s", hashPrefix, ak, ret["Signature"]) } + headers[HEADER_AUTH_CAMEL] = []string{authorization} } return } -func (obsClient ObsClient) prepareHeaders(headers map[string][]string, hostName string, isV4 bool) bool { +func prepareHostAndDate(headers map[string][]string, hostName string, isV4 bool) { headers[HEADER_HOST_CAMEL] = []string{hostName} if date, ok := headers[HEADER_DATE_AMZ]; ok { flag := false @@ -88,49 +183,9 @@ func (obsClient ObsClient) prepareHeaders(headers map[string][]string, hostName delete(headers, HEADER_DATE_AMZ) } } - if _, ok := headers[HEADER_DATE_CAMEL]; !ok { headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(time.Now().UTC())} } - - if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" { - doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") - return true - } - - if obsClient.conf.securityProvider.securityToken != "" { - headers[HEADER_STS_TOKEN_AMZ] = []string{obsClient.conf.securityProvider.securityToken} - } - return false -} - -func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string, - headers map[string][]string, hostName string) (requestUrl string, err error) { - - requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params) - parsedRequestUrl, err := url.Parse(requestUrl) - if err != nil { - return "", err - } - encodeHeaders(headers) - - if hostName == "" { - hostName = parsedRequestUrl.Host - } - - isV4 := obsClient.conf.signature == SignatureV4 - - skipAuth := obsClient.prepareHeaders(headers, hostName, isV4) - - if !skipAuth { - if isV4 { - headers[HEADER_CONTENT_SHA256_AMZ] = []string{EMPTY_CONTENT_SHA256} - err = obsClient.v4Auth(method, canonicalizedUrl, parsedRequestUrl.RawQuery, headers) - } else { - err = obsClient.v2Auth(method, canonicalizedUrl, headers) - } - } - return } func encodeHeaders(headers map[string][]string) { @@ -142,7 +197,7 @@ func encodeHeaders(headers map[string][]string) { } } -func attachHeaders(headers map[string][]string) string { +func attachHeaders(headers map[string][]string, isObs bool) string { length := len(headers) _headers := make(map[string][]string, length) keys := make([]string, 0, length) @@ -150,7 +205,11 @@ func attachHeaders(headers map[string][]string) string { for key, value := range headers { _key := strings.ToLower(strings.TrimSpace(key)) if _key != "" { - if _key == "content-md5" || _key == "content-type" || _key == "date" || strings.HasPrefix(_key, HEADER_PREFIX) { + prefixheader := HEADER_PREFIX + if isObs { + prefixheader = HEADER_PREFIX_OBS + } + if _key == "content-md5" || _key == "content-type" || _key == "date" || strings.HasPrefix(_key, prefixheader) { keys = append(keys, _key) _headers[_key] = value } @@ -165,14 +224,39 @@ func attachHeaders(headers map[string][]string) string { keys = append(keys, interestedHeader) } } + dateCamelHeader := PARAM_DATE_AMZ_CAMEL + dataHeader := HEADER_DATE_AMZ + if isObs { + dateCamelHeader = PARAM_DATE_OBS_CAMEL + dataHeader = HEADER_DATE_OBS + } + if _, ok := _headers[HEADER_DATE_CAMEL]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } else if _, ok := _headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } sort.Strings(keys) stringToSign := make([]string, 0, len(keys)) for _, key := range keys { var value string - if strings.HasPrefix(key, HEADER_PREFIX) { - if strings.HasPrefix(key, HEADER_PREFIX_META) { + prefixHeader := HEADER_PREFIX + prefixMetaHeader := HEADER_PREFIX_META + if isObs { + prefixHeader = HEADER_PREFIX_OBS + prefixMetaHeader = HEADER_PREFIX_META_OBS + } + if strings.HasPrefix(key, prefixHeader) { + if strings.HasPrefix(key, prefixMetaHeader) { for index, v := range _headers[key] { value += strings.TrimSpace(v) if index != len(_headers[key])-1 { @@ -191,18 +275,15 @@ func attachHeaders(headers map[string][]string) string { return strings.Join(stringToSign, "\n") } -func getV2StringToSign(method, canonicalizedUrl string, headers map[string][]string) string { - stringToSign := strings.Join([]string{method, "\n", attachHeaders(headers), "\n", canonicalizedUrl}, "") +func getV2StringToSign(method, canonicalizedUrl string, headers map[string][]string, isObs bool) string { + stringToSign := strings.Join([]string{method, "\n", attachHeaders(headers, isObs), "\n", canonicalizedUrl}, "") doLog(LEVEL_DEBUG, "The v2 auth stringToSign:\n%s", stringToSign) return stringToSign } -func (obsClient ObsClient) v2Auth(method, canonicalizedUrl string, headers map[string][]string) error { - stringToSign := getV2StringToSign(method, canonicalizedUrl, headers) - signature := Base64Encode(HmacSha1([]byte(obsClient.conf.securityProvider.sk), []byte(stringToSign))) - - headers[HEADER_AUTH_CAMEL] = []string{fmt.Sprintf("%s %s:%s", V2_HASH_PREFIX, obsClient.conf.securityProvider.ak, signature)} - return nil +func v2Auth(ak, sk, method, canonicalizedUrl string, headers map[string][]string, isObs bool) map[string]string { + stringToSign := getV2StringToSign(method, canonicalizedUrl, headers, isObs) + return map[string]string{"Signature": Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign)))} } func getScope(region, shortDate string) string { @@ -215,7 +296,6 @@ func getCredential(ak, region, shortDate string) (string, string) { } func getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payload string, signedHeaders []string, headers map[string][]string) string { - canonicalRequest := make([]string, 0, 10+len(signedHeaders)*4) canonicalRequest = append(canonicalRequest, method) canonicalRequest = append(canonicalRequest, "\n") @@ -239,6 +319,7 @@ func getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payl canonicalRequest = append(canonicalRequest, payload) _canonicalRequest := strings.Join(canonicalRequest, "") + doLog(LEVEL_DEBUG, "The v4 auth canonicalRequest:\n%s", _canonicalRequest) stringToSign := make([]string, 0, 7) @@ -281,9 +362,37 @@ func getSignature(stringToSign, sk, region, shortDate string) string { return Hex(HmacSha256(key, []byte(stringToSign))) } -func (obsClient ObsClient) v4Auth(method, canonicalizedUrl, queryUrl string, headers map[string][]string) error { - t, err := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) - if err != nil { +func V4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl string, headers map[string][]string) map[string]string { + return v4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl, headers) +} + +func v4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl string, headers map[string][]string) map[string]string { + var t time.Time + if val, ok := headers[HEADER_DATE_AMZ]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[HEADER_DATE_CAMEL]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else { t = time.Now().UTC() } shortDate := t.Format(SHORT_DATE_FORMAT) @@ -291,11 +400,19 @@ func (obsClient ObsClient) v4Auth(method, canonicalizedUrl, queryUrl string, hea signedHeaders, _headers := getSignedHeaders(headers) - credential, scope := getCredential(obsClient.conf.securityProvider.ak, obsClient.conf.region, shortDate) + credential, scope := getCredential(ak, region, shortDate) + + payload := EMPTY_CONTENT_SHA256 + if val, ok := headers[HEADER_CONTENT_SHA256_AMZ]; ok { + payload = val[0] + } + stringToSign := getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payload, signedHeaders, _headers) - stringToSign := getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, EMPTY_CONTENT_SHA256, signedHeaders, _headers) + signature := getSignature(stringToSign, sk, region, shortDate) - signature := getSignature(stringToSign, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) - headers[HEADER_AUTH_CAMEL] = []string{fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, credential, strings.Join(signedHeaders, ";"), signature)} - return nil + ret := make(map[string]string, 3) + ret["Credential"] = credential + ret["SignedHeaders"] = strings.Join(signedHeaders, ";") + ret["Signature"] = signature + return ret } diff --git a/openstack/obs/client.go b/openstack/obs/client.go index a222e7510..cf4aaf283 100644 --- a/openstack/obs/client.go +++ b/openstack/obs/client.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -13,12 +25,12 @@ import ( type ObsClient struct { conf *config httpClient *http.Client - transport *http.Transport } func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) { conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, endpoint: endpoint} conf.maxRetryCount = -1 + conf.maxRedirectCount = -1 for _, configurer := range configurers { configurer(conf) } @@ -26,23 +38,24 @@ func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) if err := conf.initConfigWithDefault(); err != nil { return nil, err } - - transport, err := conf.getTransport() + err := conf.getTransport() if err != nil { return nil, err } - info := make([]string, 3) - info[0] = fmt.Sprintf("[OBS SDK Version=%s", obs_sdk_version) - info[1] = fmt.Sprintf("Endpoint=%s", conf.endpoint) - accessMode := "Virtual Hosting" - if conf.pathStyle { - accessMode = "Path" + if isWarnLogEnabled() { + info := make([]string, 3) + info[0] = fmt.Sprintf("[OBS SDK Version=%s", obs_sdk_version) + info[1] = fmt.Sprintf("Endpoint=%s", conf.endpoint) + accessMode := "Virtual Hosting" + if conf.pathStyle { + accessMode = "Path" + } + info[2] = fmt.Sprintf("Access Mode=%s]", accessMode) + doLog(LEVEL_WARN, strings.Join(info, "];[")) } - info[2] = fmt.Sprintf("Access Mode=%s]", accessMode) - doLog(LEVEL_WARN, strings.Join(info, "];[")) doLog(LEVEL_DEBUG, "Create obsclient with config:\n%s\n", conf) - obsClient := &ObsClient{conf: conf, httpClient: &http.Client{Transport: transport, CheckRedirect: checkRedirectFunc}, transport: transport} + obsClient := &ObsClient{conf: conf, httpClient: &http.Client{Transport: conf.transport, CheckRedirect: checkRedirectFunc}} return obsClient, nil } @@ -52,17 +65,12 @@ func (obsClient ObsClient) Refresh(ak, sk, securityToken string) { } func (obsClient ObsClient) Close() { - obsClient.transport.CloseIdleConnections() - obsClient.transport = nil obsClient.httpClient = nil + obsClient.conf.transport.CloseIdleConnections() obsClient.conf = nil SyncLog() } -func (obsClient ObsClient) GetEndpoint() string { - return obsClient.conf.endpoint -} - func (obsClient ObsClient) ListBuckets(input *ListBucketsInput) (output *ListBucketsOutput, err error) { if input == nil { input = &ListBucketsInput{} @@ -107,15 +115,39 @@ func (obsClient ObsClient) SetBucketStoragePolicy(input *SetBucketStoragePolicyI } return } +func (obsClient ObsClient) getBucketStoragePolicyS3(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputS3 *getBucketStoragePolicyOutputS3 + outputS3 = &getBucketStoragePolicyOutputS3{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), outputS3) + if err != nil { + output = nil + return + } + output.BaseModel = outputS3.BaseModel + output.StorageClass = fmt.Sprintf("%s", outputS3.StorageClass) + return +} -func (obsClient ObsClient) GetBucketStoragePolicy(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { +func (obsClient ObsClient) getBucketStoragePolicyObs(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { output = &GetBucketStoragePolicyOutput{} - err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), output) + var outputObs *getBucketStoragePolicyOutputObs + outputObs = &getBucketStoragePolicyOutputObs{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageClass), outputObs) if err != nil { output = nil + return } + output.BaseModel = outputObs.BaseModel + output.StorageClass = outputObs.StorageClass return } +func (obsClient ObsClient) GetBucketStoragePolicy(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketStoragePolicyObs(bucketName) + } + return obsClient.getBucketStoragePolicyS3(bucketName) +} func (obsClient ObsClient) ListObjects(input *ListObjectsInput) (output *ListObjectsOutput, err error) { if input == nil { @@ -202,6 +234,17 @@ func (obsClient ObsClient) GetBucketMetadata(input *GetBucketMetadataInput) (out return } +func (obsClient ObsClient) SetObjectMetadata(input *SetObjectMetadataInput) (output *SetObjectMetadataOutput, err error) { + output = &SetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("SetObjectMetadata", HTTP_PUT, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseSetObjectMetadataOutput(output) + } + return +} + func (obsClient ObsClient) GetBucketStorageInfo(bucketName string) (output *GetBucketStorageInfoOutput, err error) { output = &GetBucketStorageInfoOutput{} err = obsClient.doActionWithBucket("GetBucketStorageInfo", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageInfo), output) @@ -211,14 +254,38 @@ func (obsClient ObsClient) GetBucketStorageInfo(bucketName string) (output *GetB return } -func (obsClient ObsClient) GetBucketLocation(bucketName string) (output *GetBucketLocationOutput, err error) { +func (obsClient ObsClient) getBucketLocationS3(bucketName string) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputS3 *getBucketLocationOutputS3 + outputS3 = &getBucketLocationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputS3) + if err != nil { + output = nil + } else { + output.BaseModel = outputS3.BaseModel + output.Location = outputS3.Location + } + return +} +func (obsClient ObsClient) getBucketLocationObs(bucketName string) (output *GetBucketLocationOutput, err error) { output = &GetBucketLocationOutput{} - err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), output) + var outputObs *getBucketLocationOutputObs + outputObs = &getBucketLocationOutputObs{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputObs) if err != nil { output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Location = outputObs.Location } return } +func (obsClient ObsClient) GetBucketLocation(bucketName string) (output *GetBucketLocationOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketLocationObs(bucketName) + } + return obsClient.getBucketLocationS3(bucketName) +} func (obsClient ObsClient) SetBucketAcl(input *SetBucketAclInput) (output *BaseModel, err error) { if input == nil { @@ -231,9 +298,36 @@ func (obsClient ObsClient) SetBucketAcl(input *SetBucketAclInput) (output *BaseM } return } - +func (obsClient ObsClient) getBucketAclObs(bucketName string) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + var outputObs *getBucketAclOutputObs + outputObs = &getBucketAclOutputObs{} + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), outputObs) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Owner = outputObs.Owner + output.Grants = make([]Grant, 0, len(outputObs.Grants)) + for _, valGrant := range outputObs.Grants { + tempOutput := Grant{} + tempOutput.Delivered = valGrant.Delivered + tempOutput.Permission = valGrant.Permission + tempOutput.Grantee.DisplayName = valGrant.Grantee.DisplayName + tempOutput.Grantee.ID = valGrant.Grantee.ID + tempOutput.Grantee.Type = valGrant.Grantee.Type + tempOutput.Grantee.URI = GroupAllUsers + + output.Grants = append(output.Grants, tempOutput) + } + } + return +} func (obsClient ObsClient) GetBucketAcl(bucketName string) (output *GetBucketAclOutput, err error) { output = &GetBucketAclOutput{} + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketAclObs(bucketName) + } err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), output) if err != nil { output = nil @@ -446,6 +540,9 @@ func (obsClient ObsClient) SetBucketNotification(input *SetBucketNotificationInp } func (obsClient ObsClient) GetBucketNotification(bucketName string) (output *GetBucketNotificationOutput, err error) { + if obsClient.conf.signature != SignatureObs { + return obsClient.getBucketNotificationS3(bucketName) + } output = &GetBucketNotificationOutput{} err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), output) if err != nil { @@ -454,6 +551,33 @@ func (obsClient ObsClient) GetBucketNotification(bucketName string) (output *Get return } +func (obsClient ObsClient) getBucketNotificationS3(bucketName string) (output *GetBucketNotificationOutput, err error) { + outputS3 := &getBucketNotificationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), outputS3) + if err != nil { + return nil, err + } + + output = &GetBucketNotificationOutput{} + output.BaseModel = outputS3.BaseModel + topicConfigurations := make([]TopicConfiguration, 0, len(outputS3.TopicConfigurations)) + for _, topicConfigurationS3 := range outputS3.TopicConfigurations { + topicConfiguration := TopicConfiguration{} + topicConfiguration.ID = topicConfigurationS3.ID + topicConfiguration.Topic = topicConfigurationS3.Topic + topicConfiguration.FilterRules = topicConfigurationS3.FilterRules + + events := make([]EventType, 0, len(topicConfigurationS3.Events)) + for _, event := range topicConfigurationS3.Events { + events = append(events, ParseStringToEventType(event)) + } + topicConfiguration.Events = events + topicConfigurations = append(topicConfigurations, topicConfiguration) + } + output.TopicConfigurations = topicConfigurations + return +} + func (obsClient ObsClient) DeleteObject(input *DeleteObjectInput) (output *DeleteObjectOutput, err error) { if input == nil { return nil, errors.New("DeleteObjectInput is nil") @@ -554,7 +678,7 @@ func (obsClient ObsClient) PutObject(input *PutObjectInput) (output *PutObjectOu } if input.ContentType == "" && input.Key != "" { - if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok { + if contentType, ok := mime_types[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { input.ContentType = contentType } } @@ -616,9 +740,9 @@ func (obsClient ObsClient) PutFile(input *PutFileInput) (output *PutObjectOutput _input.Body = body if _input.ContentType == "" && _input.Key != "" { - if contentType, ok := mime_types[_input.Key[strings.LastIndex(_input.Key, ".")+1:]]; ok { + if contentType, ok := mime_types[strings.ToLower(_input.Key[strings.LastIndex(_input.Key, ".")+1:])]; ok { _input.ContentType = contentType - } else if contentType, ok := mime_types[sourceFile[strings.LastIndex(sourceFile, ".")+1:]]; ok { + } else if contentType, ok := mime_types[strings.ToLower(sourceFile[strings.LastIndex(sourceFile, ".")+1:])]; ok { _input.ContentType = contentType } } @@ -676,7 +800,7 @@ func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploa } if input.ContentType == "" && input.Key != "" { - if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok { + if contentType, ok := mime_types[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { input.ContentType = contentType } } @@ -691,20 +815,32 @@ func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploa return } -func (obsClient ObsClient) UploadPart(input *UploadPartInput) (output *UploadPartOutput, err error) { - if input == nil { +func (obsClient ObsClient) UploadPart(_input *UploadPartInput) (output *UploadPartOutput, err error) { + if _input == nil { return nil, errors.New("UploadPartInput is nil") } - if input.UploadId == "" { + if _input.UploadId == "" { return nil, errors.New("UploadId is empty") } + input := &UploadPartInput{} + input.Bucket = _input.Bucket + input.Key = _input.Key + input.PartNumber = _input.PartNumber + input.UploadId = _input.UploadId + input.ContentMD5 = _input.ContentMD5 + input.SourceFile = _input.SourceFile + input.Offset = _input.Offset + input.PartSize = _input.PartSize + input.SseHeader = _input.SseHeader + input.Body = _input.Body + output = &UploadPartOutput{} var repeatable bool if input.Body != nil { _, repeatable = input.Body.(*strings.Reader) - if input.PartSize > 0 { + if _, ok := input.Body.(*readerWrapper); !ok && input.PartSize > 0 { input.Body = &readerWrapper{reader: input.Body, totalCount: input.PartSize} } } else if sourceFile := strings.TrimSpace(input.SourceFile); sourceFile != "" { @@ -730,7 +866,9 @@ func (obsClient ObsClient) UploadPart(input *UploadPartInput) (output *UploadPar input.PartSize = fileSize - input.Offset } fileReaderWrapper.totalCount = input.PartSize - fd.Seek(input.Offset, 0) + if _, err = fd.Seek(input.Offset, io.SeekStart); err != nil { + return nil, err + } input.Body = fileReaderWrapper repeatable = true } diff --git a/openstack/obs/conf.go b/openstack/obs/conf.go index a5f0a5319..78b777cbd 100644 --- a/openstack/obs/conf.go +++ b/openstack/obs/conf.go @@ -1,6 +1,19 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -43,15 +56,19 @@ type config struct { maxConnsPerHost int sslVerify bool pemCerts []byte + transport *http.Transport + ctx context.Context + cname bool + maxRedirectCount int } func (conf config) String() string { return fmt.Sprintf("[endpoint:%s, signature:%s, pathStyle:%v, region:%s"+ "\nconnectTimeout:%d, socketTimeout:%dheaderTimeout:%d, idleConnTimeout:%d"+ - "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, proxyUrl:%s]", + "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, proxyUrl:%s, maxRedirectCount:%d]", conf.endpoint, conf.signature, conf.pathStyle, conf.region, conf.connectTimeout, conf.socketTimeout, conf.headerTimeout, conf.idleConnTimeout, - conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.proxyUrl, + conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.proxyUrl, conf.maxRedirectCount, ) } @@ -134,6 +151,30 @@ func WithSecurityToken(securityToken string) configurer { } } +func WithHttpTransport(transport *http.Transport) configurer { + return func(conf *config) { + conf.transport = transport + } +} + +func WithRequestContext(ctx context.Context) configurer { + return func(conf *config) { + conf.ctx = ctx + } +} + +func WithCustomDomainName(cname bool) configurer { + return func(conf *config) { + conf.cname = cname + } +} + +func WithMaxRedirectCount(maxRedirectCount int) configurer { + return func(conf *config) { + conf.maxRedirectCount = maxRedirectCount + } +} + func (conf *config) initConfigWithDefault() error { conf.securityProvider.ak = strings.TrimSpace(conf.securityProvider.ak) conf.securityProvider.sk = strings.TrimSpace(conf.securityProvider.sk) @@ -183,6 +224,10 @@ func (conf *config) initConfigWithDefault() error { } } + if IsIP(urlHolder.host) { + conf.pathStyle = true + } + conf.urlHolder = urlHolder conf.region = strings.TrimSpace(conf.region) @@ -216,71 +261,110 @@ func (conf *config) initConfigWithDefault() error { conf.maxConnsPerHost = DEFAULT_MAX_CONN_PER_HOST } + if conf.maxRedirectCount < 0 { + conf.maxRedirectCount = DEFAULT_MAX_REDIRECT_COUNT + } + conf.proxyUrl = strings.TrimSpace(conf.proxyUrl) return nil } -func (conf *config) getTransport() (*http.Transport, error) { - transport := &http.Transport{ - Dial: func(network, addr string) (net.Conn, error) { - conn, err := net.DialTimeout(network, addr, time.Second*time.Duration(conf.connectTimeout)) +func (conf *config) getTransport() error { + if conf.transport == nil { + conf.transport = &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(network, addr, time.Second*time.Duration(conf.connectTimeout)) + if err != nil { + return nil, err + } + return getConnDelegate(conn, conf.socketTimeout, conf.finalTimeout), nil + }, + MaxIdleConns: conf.maxConnsPerHost, + MaxIdleConnsPerHost: conf.maxConnsPerHost, + ResponseHeaderTimeout: time.Second * time.Duration(conf.headerTimeout), + IdleConnTimeout: time.Second * time.Duration(conf.idleConnTimeout), + } + + if conf.proxyUrl != "" { + proxyUrl, err := url.Parse(conf.proxyUrl) if err != nil { - return nil, err + return err } - return getConnDelegate(conn, conf.socketTimeout, conf.finalTimeout), nil - }, - MaxIdleConns: conf.maxConnsPerHost, - MaxIdleConnsPerHost: conf.maxConnsPerHost, - ResponseHeaderTimeout: time.Second * time.Duration(conf.headerTimeout), - IdleConnTimeout: time.Second * time.Duration(conf.idleConnTimeout), - } - - if conf.proxyUrl != "" { - proxyUrl, err := url.Parse(conf.proxyUrl) - if err != nil { - return nil, err + conf.transport.Proxy = http.ProxyURL(proxyUrl) + } + + tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify} + if conf.sslVerify && conf.pemCerts != nil { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(conf.pemCerts) + tlsConfig.RootCAs = pool } - transport.Proxy = http.ProxyURL(proxyUrl) - } - tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify} - if conf.sslVerify && conf.pemCerts != nil { - pool := x509.NewCertPool() - pool.AppendCertsFromPEM(conf.pemCerts) - tlsConfig.RootCAs = pool + conf.transport.TLSClientConfig = tlsConfig } - transport.TLSClientConfig = tlsConfig - return transport, nil + return nil } func checkRedirectFunc(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } -func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string) (requestUrl string, canonicalizedUrl string) { +func DummyQueryEscape(s string) string { + return s +} + +func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string, escape bool) (requestUrl string, canonicalizedUrl string) { urlHolder := conf.urlHolder - - if bucketName == "" { + if conf.cname { requestUrl = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) - canonicalizedUrl = "/" + if conf.signature == "v4" { + canonicalizedUrl = "/" + } else { + canonicalizedUrl = "/" + urlHolder.host + "/" + } } else { - if conf.pathStyle { - requestUrl = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) - canonicalizedUrl = "/" + bucketName + if bucketName == "" { + requestUrl = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + canonicalizedUrl = "/" } else { - requestUrl = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) - if conf.signature == "v2" { - canonicalizedUrl = "/" + bucketName + "/" + if conf.pathStyle { + requestUrl = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) + canonicalizedUrl = "/" + bucketName } else { - canonicalizedUrl = "/" + requestUrl = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) + if conf.signature == "v2" || conf.signature == "OBS" { + canonicalizedUrl = "/" + bucketName + "/" + } else { + canonicalizedUrl = "/" + } } } } + var escapeFunc func(s string) string + if escape { + escapeFunc = url.QueryEscape + } else { + escapeFunc = DummyQueryEscape + } if objectKey != "" { - encodeObjectKey := url.QueryEscape(objectKey) + var encodeObjectKey string + if escape { + tempKey := []rune(objectKey) + result := make([]string, 0, len(tempKey)) + for _, value := range tempKey { + if string(value) == "/" { + result = append(result, string(value)) + } else { + result = append(result, url.QueryEscape(string(value))) + } + } + encodeObjectKey = strings.Join(result, "") + } else { + encodeObjectKey = escapeFunc(objectKey) + } requestUrl += "/" + encodeObjectKey if !strings.HasSuffix(canonicalizedUrl, "/") { canonicalizedUrl += "/" @@ -294,6 +378,7 @@ func (conf *config) formatUrls(bucketName, objectKey string, params map[string]s } sort.Strings(keys) i := 0 + for index, key := range keys { if index == 0 { requestUrl += "?" @@ -304,7 +389,6 @@ func (conf *config) formatUrls(bucketName, objectKey string, params map[string]s requestUrl += _key _value := params[key] - if conf.signature == "v4" { requestUrl += "=" + url.QueryEscape(_value) } else { @@ -314,7 +398,15 @@ func (conf *config) formatUrls(bucketName, objectKey string, params map[string]s } else { _value = "" } - if _, ok := allowed_resource_parameter_names[strings.ToLower(key)]; ok { + lowerKey := strings.ToLower(key) + _, ok := allowed_resource_parameter_names[lowerKey] + prefixHeader := HEADER_PREFIX + isObs := conf.signature == SignatureObs + if isObs { + prefixHeader = HEADER_PREFIX_OBS + } + ok = ok || strings.HasPrefix(lowerKey, prefixHeader) + if ok { if i == 0 { canonicalizedUrl += "?" } else { diff --git a/openstack/obs/const.go b/openstack/obs/const.go index 24716162b..63f952b85 100644 --- a/openstack/obs/const.go +++ b/openstack/obs/const.go @@ -1,53 +1,84 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs const ( - obs_sdk_version = "2.2.1" - USER_AGENT = "obs-sdk-go/" + obs_sdk_version - HEADER_PREFIX = "x-amz-" - HEADER_PREFIX_META = "x-amz-meta-" - HEADER_DATE_AMZ = "x-amz-date" - HEADER_STS_TOKEN_AMZ = "x-amz-security-token" - - PREFIX_META = "meta-" - - HEADER_CONTENT_SHA256_AMZ = "x-amz-content-sha256" - HEADER_ACL_AMZ = "x-amz-acl" - HEADER_COPY_SOURCE_AMZ = "x-amz-copy-source" - HEADER_COPY_SOURCE_RANGE_AMZ = "x-amz-copy-source-range" - HEADER_RANGE = "Range" - HEADER_STORAGE_CLASS = "x-default-storage-class" - HEADER_REQUEST_ID = "request-id" - HEADER_BUCKET_REGION = "bucket-region" - HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN = "access-control-allow-origin" - HEADER_ACCESS_CONRTOL_ALLOW_HEADERS = "access-control-allow-headers" - HEADER_ACCESS_CONRTOL_MAX_AGE = "access-control-max-age" - HEADER_ACCESS_CONRTOL_ALLOW_METHODS = "access-control-allow-methods" - HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS = "access-control-expose-headers" - HEADER_VERSION_ID = "version-id" - HEADER_COPY_SOURCE_VERSION_ID = "copy-source-version-id" - HEADER_DELETE_MARKER = "delete-marker" - HEADER_WEBSITE_REDIRECT_LOCATION = "website-redirect-location" - HEADER_WEBSITE_REDIRECT_LOCATION_AMZ = "x-amz-website-redirect-location" - HEADER_METADATA_DIRECTIVE_AMZ = "x-amz-metadata-directive" - HEADER_EXPIRATION = "expiration" - HEADER_RESTORE = "restore" - HEADER_STORAGE_CLASS2 = "storage-class" - HEADER_STORAGE_CLASS2_AMZ = "x-amz-storage-class" - HEADER_CONTENT_LENGTH = "content-length" - HEADER_CONTENT_TYPE = "content-type" - HEADER_CONTENT_LANGUAGE = "content-language" - HEADER_EXPIRES = "expires" - HEADER_CACHE_CONTROL = "cache-control" - HEADER_CONTENT_DISPOSITION = "content-disposition" - HEADER_CONTENT_ENCODING = "content-encoding" + obs_sdk_version = "3.19.11" + USER_AGENT = "obs-sdk-go/" + obs_sdk_version + HEADER_PREFIX = "x-amz-" + HEADER_PREFIX_META = "x-amz-meta-" + HEADER_PREFIX_OBS = "x-obs-" + HEADER_PREFIX_META_OBS = "x-obs-meta-" + HEADER_DATE_AMZ = "x-amz-date" + HEADER_DATE_OBS = "x-obs-date" + HEADER_STS_TOKEN_AMZ = "x-amz-security-token" + HEADER_STS_TOKEN_OBS = "x-obs-security-token" + HEADER_ACCESSS_KEY_AMZ = "AWSAccessKeyId" + PREFIX_META = "meta-" + + HEADER_CONTENT_SHA256_AMZ = "x-amz-content-sha256" + HEADER_ACL_AMZ = "x-amz-acl" + HEADER_ACL_OBS = "x-obs-acl" + HEADER_ACL = "acl" + HEADER_LOCATION_AMZ = "location" + HEADER_BUCKET_LOCATION_OBS = "bucket-location" + HEADER_COPY_SOURCE = "copy-source" + HEADER_COPY_SOURCE_RANGE = "copy-source-range" + HEADER_RANGE = "Range" + HEADER_STORAGE_CLASS = "x-default-storage-class" + HEADER_STORAGE_CLASS_OBS = "x-obs-storage-class" + HEADER_VERSION_OBS = "version" + HEADER_GRANT_READ_OBS = "grant-read" + HEADER_GRANT_WRITE_OBS = "grant-write" + HEADER_GRANT_READ_ACP_OBS = "grant-read-acp" + HEADER_GRANT_WRITE_ACP_OBS = "grant-write-acp" + HEADER_GRANT_FULL_CONTROL_OBS = "grant-full-control" + HEADER_GRANT_READ_DELIVERED_OBS = "grant-read-delivered" + HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS = "grant-full-control-delivered" + HEADER_REQUEST_ID = "request-id" + HEADER_BUCKET_REGION = "bucket-region" + HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN = "access-control-allow-origin" + HEADER_ACCESS_CONRTOL_ALLOW_HEADERS = "access-control-allow-headers" + HEADER_ACCESS_CONRTOL_MAX_AGE = "access-control-max-age" + HEADER_ACCESS_CONRTOL_ALLOW_METHODS = "access-control-allow-methods" + HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS = "access-control-expose-headers" + HEADER_EPID_HEADERS = "epid" + HEADER_VERSION_ID = "version-id" + HEADER_COPY_SOURCE_VERSION_ID = "copy-source-version-id" + HEADER_DELETE_MARKER = "delete-marker" + HEADER_WEBSITE_REDIRECT_LOCATION = "website-redirect-location" + HEADER_METADATA_DIRECTIVE = "metadata-directive" + HEADER_EXPIRATION = "expiration" + HEADER_EXPIRES_OBS = "x-obs-expires" + HEADER_RESTORE = "restore" + HEADER_OBJECT_TYPE = "object-type" + HEADER_NEXT_APPEND_POSITION = "next-append-position" + HEADER_STORAGE_CLASS2 = "storage-class" + HEADER_CONTENT_LENGTH = "content-length" + HEADER_CONTENT_TYPE = "content-type" + HEADER_CONTENT_LANGUAGE = "content-language" + HEADER_EXPIRES = "expires" + HEADER_CACHE_CONTROL = "cache-control" + HEADER_CONTENT_DISPOSITION = "content-disposition" + HEADER_CONTENT_ENCODING = "content-encoding" HEADER_ETAG = "etag" HEADER_LASTMODIFIED = "last-modified" - HEADER_COPY_SOURCE_IF_MATCH_AMZ = "x-amz-copy-source-if-match" - HEADER_COPY_SOURCE_IF_NONE_MATCH_AMZ = "x-amz-copy-source-if-none-match" - HEADER_COPY_SOURCE_IF_MODIFIED_SINCE_AMZ = "x-amz-copy-source-if-modified-since" - HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE_AMZ = "x-amz-copy-source-if-unmodified-since" + HEADER_COPY_SOURCE_IF_MATCH = "copy-source-if-match" + HEADER_COPY_SOURCE_IF_NONE_MATCH = "copy-source-if-none-match" + HEADER_COPY_SOURCE_IF_MODIFIED_SINCE = "copy-source-if-modified-since" + HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE = "copy-source-if-unmodified-since" HEADER_IF_MATCH = "If-Match" HEADER_IF_NONE_MATCH = "If-None-Match" @@ -58,19 +89,19 @@ const ( HEADER_SSEC_KEY = "server-side-encryption-customer-key" HEADER_SSEC_KEY_MD5 = "server-side-encryption-customer-key-MD5" - HEADER_SSEKMS_ENCRYPTION = "server-side-encryption" - HEADER_SSEKMS_KEY = "server-side-encryption-aws-kms-key-id" + HEADER_SSEKMS_ENCRYPTION = "server-side-encryption" + HEADER_SSEKMS_KEY = "server-side-encryption-aws-kms-key-id" + HEADER_SSEKMS_ENCRYPT_KEY_OBS = "server-side-encryption-kms-key-id" + + HEADER_SSEC_COPY_SOURCE_ENCRYPTION = "copy-source-server-side-encryption-customer-algorithm" + HEADER_SSEC_COPY_SOURCE_KEY = "copy-source-server-side-encryption-customer-key" + HEADER_SSEC_COPY_SOURCE_KEY_MD5 = "copy-source-server-side-encryption-customer-key-MD5" - HEADER_SSEC_ENCRYPTION_AMZ = "x-amz-server-side-encryption-customer-algorithm" - HEADER_SSEC_KEY_AMZ = "x-amz-server-side-encryption-customer-key" - HEADER_SSEC_KEY_MD5_AMZ = "x-amz-server-side-encryption-customer-key-MD5" + HEADER_SSEKMS_KEY_AMZ = "x-amz-server-side-encryption-aws-kms-key-id" - HEADER_SSEC_COPY_SOURCE_ENCRYPTION_AMZ = "x-amz-copy-source-server-side-encryption-customer-algorithm" - HEADER_SSEC_COPY_SOURCE_KEY_AMZ = "x-amz-copy-source-server-side-encryption-customer-key" - HEADER_SSEC_COPY_SOURCE_KEY_MD5_AMZ = "x-amz-copy-source-server-side-encryption-customer-key-MD5" + HEADER_SSEKMS_KEY_OBS = "x-obs-server-side-encryption-kms-key-id" - HEADER_SSEKMS_ENCRYPTION_AMZ = "x-amz-server-side-encryption" - HEADER_SSEKMS_KEY_AMZ = "x-amz-server-side-encryption-aws-kms-key-id" + HEADER_SUCCESS_ACTION_REDIRECT = "success_action_redirect" HEADER_DATE_CAMEL = "Date" HEADER_HOST_CAMEL = "Host" @@ -83,6 +114,11 @@ const ( HEADER_USER_AGENT_CAMEL = "User-Agent" HEADER_ORIGIN_CAMEL = "Origin" HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL = "Access-Control-Request-Headers" + HEADER_CACHE_CONTROL_CAMEL = "Cache-Control" + HEADER_CONTENT_DISPOSITION_CAMEL = "Content-Disposition" + HEADER_CONTENT_ENCODING_CAMEL = "Content-Encoding" + HEADER_CONTENT_LANGUAGE_CAMEL = "Content-Language" + HEADER_EXPIRES_CAMEL = "Expires" PARAM_VERSION_ID = "versionId" PARAM_RESPONSE_CONTENT_TYPE = "response-content-type" @@ -96,6 +132,7 @@ const ( PARAM_ALGORITHM_AMZ_CAMEL = "X-Amz-Algorithm" PARAM_CREDENTIAL_AMZ_CAMEL = "X-Amz-Credential" PARAM_DATE_AMZ_CAMEL = "X-Amz-Date" + PARAM_DATE_OBS_CAMEL = "X-Obs-Date" PARAM_EXPIRES_AMZ_CAMEL = "X-Amz-Expires" PARAM_SIGNEDHEADERS_AMZ_CAMEL = "X-Amz-SignedHeaders" PARAM_SIGNATURE_AMZ_CAMEL = "X-Amz-Signature" @@ -107,6 +144,7 @@ const ( DEFAULT_HEADER_TIMEOUT = 60 DEFAULT_IDLE_CONN_TIMEOUT = 30 DEFAULT_MAX_RETRY_COUNT = 3 + DEFAULT_MAX_REDIRECT_COUNT = 3 DEFAULT_MAX_CONN_PER_HOST = 1000 EMPTY_CONTENT_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD" @@ -119,12 +157,16 @@ const ( V4_SERVICE_NAME = "s3" V4_SERVICE_SUFFIX = "aws4_request" - V2_HASH_PREFIX = "AWS" + V2_HASH_PREFIX = "AWS" + OBS_HASH_PREFIX = "OBS" + V4_HASH_PREFIX = "AWS4-HMAC-SHA256" V4_HASH_PRE = "AWS4" - DEFAULT_SSE_KMS_ENCRYPTION = "aws:kms" - DEFAULT_SSE_C_ENCRYPTION = "AES256" + DEFAULT_SSE_KMS_ENCRYPTION = "aws:kms" + DEFAULT_SSE_KMS_ENCRYPTION_OBS = "kms" + + DEFAULT_SSE_C_ENCRYPTION = "AES256" HTTP_GET = "GET" HTTP_POST = "POST" @@ -137,8 +179,9 @@ const ( type SignatureType string const ( - SignatureV2 SignatureType = "v2" - SignatureV4 SignatureType = "v4" + SignatureV2 SignatureType = "v2" + SignatureV4 SignatureType = "v4" + SignatureObs SignatureType = "OBS" ) var ( @@ -162,6 +205,7 @@ var ( "last-modified": true, "content-range": true, "x-reserved": true, + "x-reserved-indicator": true, "access-control-allow-origin": true, "access-control-allow-headers": true, "access-control-max-age": true, @@ -198,12 +242,14 @@ var ( allowed_resource_parameter_names = map[string]bool{ "acl": true, + "backtosource": true, "policy": true, "torrent": true, "logging": true, "location": true, "storageinfo": true, "quota": true, + "storageclass": true, "storagepolicy": true, "requestpayment": true, "versions": true, @@ -220,6 +266,9 @@ var ( "cors": true, "restore": true, "tagging": true, + "append": true, + "position": true, + "replication": true, "response-content-type": true, "response-content-language": true, "response-expires": true, @@ -227,104 +276,385 @@ var ( "response-content-disposition": true, "response-content-encoding": true, "x-image-process": true, + "x-oss-process": true, + "x-image-save-bucket": true, + "x-image-save-object": true, } mime_types = map[string]string{ + "001": "application/x-001", + "301": "application/x-301", + "323": "text/h323", "7z": "application/x-7z-compressed", + "906": "application/x-906", + "907": "drawing/907", + "IVF": "video/x-ivf", + "a11": "application/x-a11", "aac": "audio/x-aac", + "acp": "audio/x-mei-aac", "ai": "application/postscript", - "aif": "audio/x-aiff", - "asc": "text/plain", + "aif": "audio/aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "anv": "application/x-anv", + "apk": "application/vnd.android.package-archive", + "asa": "text/asa", "asf": "video/x-ms-asf", + "asp": "text/asp", + "asx": "video/x-ms-asf", "atom": "application/atom+xml", - "avi": "video/x-msvideo", - "bmp": "image/bmp", + "au": "audio/basic", + "avi": "video/avi", + "awf": "application/vnd.adobe.workflow", + "biz": "text/xml", + "bmp": "application/x-bmp", + "bot": "application/x-bot", "bz2": "application/x-bzip2", - "cer": "application/pkix-cert", + "c4t": "application/x-c4t", + "c90": "application/x-c90", + "cal": "application/x-cals", + "cat": "application/vnd.ms-pki.seccat", + "cdf": "application/x-netcdf", + "cdr": "application/x-cdr", + "cel": "application/x-cel", + "cer": "application/x-x509-ca-cert", + "cg4": "application/x-g4", + "cgm": "application/x-cgm", + "cit": "application/x-cit", + "class": "java/*", + "cml": "text/xml", + "cmp": "application/x-cmp", + "cmx": "application/x-cmx", + "cot": "application/x-cot", "crl": "application/pkix-crl", "crt": "application/x-x509-ca-cert", + "csi": "application/x-csi", "css": "text/css", "csv": "text/csv", "cu": "application/cu-seeme", + "cut": "application/x-cut", + "dbf": "application/x-dbf", + "dbm": "application/x-dbm", + "dbx": "application/x-dbx", + "dcd": "text/xml", + "dcx": "application/x-dcx", "deb": "application/x-debian-package", + "der": "application/x-x509-ca-cert", + "dgn": "application/x-dgn", + "dib": "application/x-dib", + "dll": "application/x-msdownload", "doc": "application/msword", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dot": "application/msword", + "drw": "application/x-drw", + "dtd": "text/xml", "dvi": "application/x-dvi", + "dwf": "application/x-dwf", + "dwg": "application/x-dwg", + "dxb": "application/x-dxb", + "dxf": "application/x-dxf", + "edn": "application/vnd.adobe.edn", + "emf": "application/x-emf", + "eml": "message/rfc822", + "ent": "text/xml", "eot": "application/vnd.ms-fontobject", + "epi": "application/x-epi", "eps": "application/postscript", "epub": "application/epub+zip", + "etd": "application/x-ebx", "etx": "text/x-setext", + "exe": "application/x-msdownload", + "fax": "image/fax", + "fdf": "application/vnd.fdf", + "fif": "application/fractals", "flac": "audio/flac", "flv": "video/x-flv", + "fo": "text/xml", + "frm": "application/x-frm", + "g4": "application/x-g4", + "gbr": "application/x-gbr", "gif": "image/gif", + "gl2": "application/x-gl2", + "gp4": "application/x-gp4", "gz": "application/gzip", + "hgl": "application/x-hgl", + "hmr": "application/x-hmr", + "hpg": "application/x-hpgl", + "hpl": "application/x-hpl", + "hqx": "application/mac-binhex40", + "hrf": "application/x-hrf", + "hta": "application/hta", + "htc": "text/x-component", "htm": "text/html", "html": "text/html", - "ico": "image/x-icon", + "htt": "text/webviewhtml", + "htx": "text/html", + "icb": "application/x-icb", + "ico": "application/x-ico", "ics": "text/calendar", + "iff": "application/x-iff", + "ig4": "application/x-g4", + "igs": "application/x-igs", + "iii": "application/x-iphone", + "img": "application/x-img", "ini": "text/plain", + "ins": "application/x-internet-signup", + "ipa": "application/vnd.iphone", "iso": "application/x-iso9660-image", + "isp": "application/x-internet-signup", "jar": "application/java-archive", + "java": "java/*", + "jfif": "image/jpeg", "jpe": "image/jpeg", "jpeg": "image/jpeg", "jpg": "image/jpeg", - "js": "text/javascript", + "js": "application/x-javascript", "json": "application/json", + "jsp": "text/html", + "la1": "audio/x-liquid-file", + "lar": "application/x-laplayer-reg", "latex": "application/x-latex", + "lavs": "audio/x-liquid-secure", + "lbm": "application/x-lbm", + "lmsff": "audio/x-la-lms", "log": "text/plain", + "ls": "application/x-javascript", + "ltr": "application/x-ltr", + "m1v": "video/x-mpeg", + "m2v": "video/x-mpeg", + "m3u": "audio/mpegurl", "m4a": "audio/mp4", + "m4e": "video/mpeg4", "m4v": "video/mp4", - "mid": "audio/midi", - "midi": "audio/midi", + "mac": "application/x-mac", + "man": "application/x-troff-man", + "math": "text/xml", + "mdb": "application/msaccess", + "mfp": "application/x-shockwave-flash", + "mht": "message/rfc822", + "mhtml": "message/rfc822", + "mi": "application/x-mi", + "mid": "audio/mid", + "midi": "audio/mid", + "mil": "application/x-mil", + "mml": "text/xml", + "mnd": "audio/x-musicnet-download", + "mns": "audio/x-musicnet-stream", + "mocha": "application/x-javascript", "mov": "video/quicktime", - "mp3": "audio/mpeg", - "mp4": "video/mp4", + "movie": "video/x-sgi-movie", + "mp1": "audio/mp1", + "mp2": "audio/mp2", + "mp2v": "video/mpeg", + "mp3": "audio/mp3", + "mp4": "video/mpeg4", "mp4a": "audio/mp4", "mp4v": "video/mp4", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpg": "video/mpeg", + "mpa": "video/x-mpg", + "mpd": "application/vnd.ms-project", + "mpe": "video/x-mpeg", + "mpeg": "video/mpg", + "mpg": "video/mpg", "mpg4": "video/mp4", + "mpga": "audio/rn-mpeg", + "mpp": "application/vnd.ms-project", + "mps": "video/x-mpeg", + "mpt": "application/vnd.ms-project", + "mpv": "video/mpg", + "mpv2": "video/mpeg", + "mpw": "application/vnd.ms-project", + "mpx": "application/vnd.ms-project", + "mtx": "text/xml", + "mxp": "application/x-mmxp", + "net": "image/pnetvue", + "nrf": "application/x-nrf", + "nws": "message/rfc822", + "odc": "text/x-ms-odc", "oga": "audio/ogg", "ogg": "audio/ogg", "ogv": "video/ogg", "ogx": "application/ogg", + "out": "application/x-out", + "p10": "application/pkcs10", + "p12": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "p7c": "application/pkcs7-mime", + "p7m": "application/pkcs7-mime", + "p7r": "application/x-pkcs7-certreqresp", + "p7s": "application/pkcs7-signature", "pbm": "image/x-portable-bitmap", + "pc5": "application/x-pc5", + "pci": "application/x-pci", + "pcl": "application/x-pcl", + "pcx": "application/x-pcx", "pdf": "application/pdf", + "pdx": "application/vnd.adobe.pdx", + "pfx": "application/x-pkcs12", + "pgl": "application/x-pgl", "pgm": "image/x-portable-graymap", + "pic": "application/x-pic", + "pko": "application/vnd.ms-pki.pko", + "pl": "application/x-perl", + "plg": "text/html", + "pls": "audio/scpls", + "plt": "application/x-plt", "png": "image/png", "pnm": "image/x-portable-anymap", - "ppm": "image/x-portable-pixmap", + "pot": "application/vnd.ms-powerpoint", + "ppa": "application/vnd.ms-powerpoint", + "ppm": "application/x-ppm", + "pps": "application/vnd.ms-powerpoint", "ppt": "application/vnd.ms-powerpoint", "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pr": "application/x-pr", + "prf": "application/pics-rules", + "prn": "application/x-prn", + "prt": "application/x-prt", "ps": "application/postscript", + "ptn": "application/x-ptn", + "pwz": "application/vnd.ms-powerpoint", "qt": "video/quicktime", + "r3t": "text/vnd.rn-realtext3d", + "ra": "audio/vnd.rn-realaudio", + "ram": "audio/x-pn-realaudio", "rar": "application/x-rar-compressed", - "ras": "image/x-cmu-raster", + "ras": "application/x-ras", + "rat": "application/rat-file", + "rdf": "text/xml", + "rec": "application/vnd.rn-recording", + "red": "application/x-red", + "rgb": "application/x-rgb", + "rjs": "application/vnd.rn-realsystem-rjs", + "rjt": "application/vnd.rn-realsystem-rjt", + "rlc": "application/x-rlc", + "rle": "application/x-rle", + "rm": "application/vnd.rn-realmedia", + "rmf": "application/vnd.adobe.rmf", + "rmi": "audio/mid", + "rmj": "application/vnd.rn-realsystem-rmj", + "rmm": "audio/x-pn-realaudio", + "rmp": "application/vnd.rn-rn_music_package", + "rms": "application/vnd.rn-realmedia-secure", + "rmvb": "application/vnd.rn-realmedia-vbr", + "rmx": "application/vnd.rn-realsystem-rmx", + "rnx": "application/vnd.rn-realplayer", + "rp": "image/vnd.rn-realpix", + "rpm": "audio/x-pn-realaudio-plugin", + "rsml": "application/vnd.rn-rsml", "rss": "application/rss+xml", - "rtf": "application/rtf", + "rt": "text/vnd.rn-realtext", + "rtf": "application/x-rtf", + "rv": "video/vnd.rn-realvideo", + "sam": "application/x-sam", + "sat": "application/x-sat", + "sdp": "application/sdp", + "sdw": "application/x-sdw", "sgm": "text/sgml", "sgml": "text/sgml", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "sit": "application/x-stuffit", + "slb": "application/x-slb", + "sld": "application/x-sld", + "slk": "drawing/x-slk", + "smi": "application/smil", + "smil": "application/smil", + "smk": "application/x-smk", + "snd": "audio/basic", + "sol": "text/plain", + "sor": "text/plain", + "spc": "application/x-pkcs7-certificates", + "spl": "application/futuresplash", + "spp": "text/xml", + "ssm": "application/streamingmedia", + "sst": "application/vnd.ms-pki.certstore", + "stl": "application/vnd.ms-pki.stl", + "stm": "text/html", + "sty": "application/x-sty", "svg": "image/svg+xml", "swf": "application/x-shockwave-flash", "tar": "application/x-tar", + "tdf": "application/x-tdf", + "tg4": "application/x-tg4", + "tga": "application/x-tga", "tif": "image/tiff", "tiff": "image/tiff", + "tld": "text/xml", + "top": "drawing/x-top", "torrent": "application/x-bittorrent", + "tsd": "text/xml", "ttf": "application/x-font-ttf", "txt": "text/plain", - "wav": "audio/x-wav", + "uin": "application/x-icq", + "uls": "text/iuls", + "vcf": "text/x-vcard", + "vda": "application/x-vda", + "vdx": "application/vnd.visio", + "vml": "text/xml", + "vpg": "application/x-vpeg005", + "vsd": "application/vnd.visio", + "vss": "application/vnd.visio", + "vst": "application/x-vst", + "vsw": "application/vnd.visio", + "vsx": "application/vnd.visio", + "vtx": "application/vnd.visio", + "vxml": "text/xml", + "wav": "audio/wav", + "wax": "audio/x-ms-wax", + "wb1": "application/x-wb1", + "wb2": "application/x-wb2", + "wb3": "application/x-wb3", + "wbmp": "image/vnd.wap.wbmp", "webm": "video/webm", + "wiz": "application/msword", + "wk3": "application/x-wk3", + "wk4": "application/x-wk4", + "wkq": "application/x-wkq", + "wks": "application/x-wks", + "wm": "video/x-ms-wm", "wma": "audio/x-ms-wma", + "wmd": "application/x-ms-wmd", + "wmf": "application/x-wmf", + "wml": "text/vnd.wap.wml", "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wmz": "application/x-ms-wmz", "woff": "application/x-font-woff", - "wsdl": "application/wsdl+xml", + "wp6": "application/x-wp6", + "wpd": "application/x-wpd", + "wpg": "application/x-wpg", + "wpl": "application/vnd.ms-wpl", + "wq1": "application/x-wq1", + "wr1": "application/x-wr1", + "wri": "application/x-wri", + "wrk": "application/x-wrk", + "ws": "application/x-ws", + "ws2": "application/x-ws", + "wsc": "text/scriptlet", + "wsdl": "text/xml", + "wvx": "video/x-ms-wvx", + "x_b": "application/x-x_b", + "x_t": "application/x-x_t", + "xap": "application/x-silverlight-app", "xbm": "image/x-xbitmap", + "xdp": "application/vnd.adobe.xdp", + "xdr": "text/xml", + "xfd": "application/vnd.adobe.xfd", + "xfdf": "application/vnd.adobe.xfdf", + "xhtml": "text/html", "xls": "application/vnd.ms-excel", "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xml": "application/xml", + "xlw": "application/x-xlw", + "xml": "text/xml", + "xpl": "audio/scpls", "xpm": "image/x-xpixmap", - "xwd": "image/x-xwindowdump", + "xq": "text/xml", + "xql": "text/xml", + "xquery": "text/xml", + "xsd": "text/xml", + "xsl": "text/xml", + "xslt": "text/xml", + "xwd": "application/x-xwd", "yaml": "text/yaml", "yml": "text/yaml", "zip": "application/zip", @@ -346,6 +676,7 @@ type SubResourceType string const ( SubResourceStoragePolicy SubResourceType = "storagePolicy" + SubResourceStorageClass SubResourceType = "storageClass" SubResourceQuota SubResourceType = "quota" SubResourceStorageInfo SubResourceType = "storageinfo" SubResourceLocation SubResourceType = "location" @@ -362,26 +693,29 @@ const ( SubResourceVersions SubResourceType = "versions" SubResourceUploads SubResourceType = "uploads" SubResourceRestore SubResourceType = "restore" + SubResourceMetadata SubResourceType = "metadata" ) type AclType string const ( - AclPrivate AclType = "private" - AclPublicRead AclType = "public-read" - AclPublicReadWrite AclType = "public-read-write" - AclAuthenticatedRead AclType = "authenticated-read" - AclBucketOwnerRead AclType = "bucket-owner-read" - AclBucketOwnerFullControl AclType = "bucket-owner-full-control" - AclLogDeliveryWrite AclType = "log-delivery-write" + AclPrivate AclType = "private" + AclPublicRead AclType = "public-read" + AclPublicReadWrite AclType = "public-read-write" + AclAuthenticatedRead AclType = "authenticated-read" + AclBucketOwnerRead AclType = "bucket-owner-read" + AclBucketOwnerFullControl AclType = "bucket-owner-full-control" + AclLogDeliveryWrite AclType = "log-delivery-write" + AclPublicReadDelivery AclType = "public-read-delivered" + AclPublicReadWriteDelivery AclType = "public-read-write-delivered" ) type StorageClassType string const ( StorageClassStandard StorageClassType = "STANDARD" - StorageClassWarm StorageClassType = "STANDARD_IA" - StorageClassCold StorageClassType = "GLACIER" + StorageClassWarm StorageClassType = "WARM" + StorageClassCold StorageClassType = "COLD" ) type PermissionType string @@ -404,9 +738,9 @@ const ( type GroupUriType string const ( - GroupAllUsers GroupUriType = "http://acs.amazonaws.com/groups/global/AllUsers" - GroupAuthenticatedUsers GroupUriType = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" - GroupLogDelivery GroupUriType = "http://acs.amazonaws.com/groups/s3/LogDelivery" + GroupAllUsers GroupUriType = "AllUsers" + GroupAuthenticatedUsers GroupUriType = "AuthenticatedUsers" + GroupLogDelivery GroupUriType = "LogDelivery" ) type VersioningStatusType string @@ -442,5 +776,20 @@ type MetadataDirectiveType string const ( CopyMetadata MetadataDirectiveType = "COPY" + ReplaceNew MetadataDirectiveType = "REPLACE_NEW" ReplaceMetadata MetadataDirectiveType = "REPLACE" ) + +type EventType string + +const ( + ObjectCreatedAll EventType = "ObjectCreated:*" + ObjectCreatedPut EventType = "ObjectCreated:Put" + ObjectCreatedPost EventType = "ObjectCreated:Post" + + ObjectCreatedCopy EventType = "ObjectCreated:Copy" + ObjectCreatedCompleteMultipartUpload EventType = "ObjectCreated:CompleteMultipartUpload" + ObjectRemovedAll EventType = "ObjectRemoved:*" + ObjectRemovedDelete EventType = "ObjectRemoved:Delete" + ObjectRemovedDeleteMarkerCreated EventType = "ObjectRemoved:DeleteMarkerCreated" +) diff --git a/openstack/obs/convert.go b/openstack/obs/convert.go index f4ab93d37..c0d5149e6 100644 --- a/openstack/obs/convert.go +++ b/openstack/obs/convert.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -16,7 +28,7 @@ func cleanHeaderPrefix(header http.Header) map[string][]string { for key, value := range header { if len(value) > 0 { key = strings.ToLower(key) - if strings.HasPrefix(key, HEADER_PREFIX) { + if strings.HasPrefix(key, HEADER_PREFIX) || strings.HasPrefix(key, HEADER_PREFIX_OBS) { key = key[len(HEADER_PREFIX):] } responseHeaders[key] = value @@ -25,13 +37,37 @@ func cleanHeaderPrefix(header http.Header) map[string][]string { return responseHeaders } +func ParseStringToEventType(value string) (ret EventType) { + switch value { + case "ObjectCreated:*", "s3:ObjectCreated:*": + ret = ObjectCreatedAll + case "ObjectCreated:Put", "s3:ObjectCreated:Put": + ret = ObjectCreatedPut + case "ObjectCreated:Post", "s3:ObjectCreated:Post": + ret = ObjectCreatedPost + case "ObjectCreated:Copy", "s3:ObjectCreated:Copy": + ret = ObjectCreatedCopy + case "ObjectCreated:CompleteMultipartUpload", "s3:ObjectCreated:CompleteMultipartUpload": + ret = ObjectCreatedCompleteMultipartUpload + case "ObjectRemoved:*", "s3:ObjectRemoved:*": + ret = ObjectRemovedAll + case "ObjectRemoved:Delete", "s3:ObjectRemoved:Delete": + ret = ObjectRemovedDelete + case "ObjectRemoved:DeleteMarkerCreated", "s3:ObjectRemoved:DeleteMarkerCreated": + ret = ObjectRemovedDeleteMarkerCreated + default: + ret = "" + } + return +} + func ParseStringToStorageClassType(value string) (ret StorageClassType) { switch value { case "STANDARD": ret = StorageClassStandard - case "STANDARD_IA": + case "STANDARD_IA", "WARM": ret = StorageClassWarm - case "GLACIER": + case "GLACIER", "COLD": ret = StorageClassCold default: ret = "" @@ -39,35 +75,74 @@ func ParseStringToStorageClassType(value string) (ret StorageClassType) { return } -func convertGrantToXml(grant Grant) string { +func convertGrantToXml(grant Grant, isObs bool, isBucket bool) string { xml := make([]string, 0, 4) - xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + if !isObs { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + } + if grant.Grantee.Type == GranteeUser { - xml = append(xml, fmt.Sprintf("%s", grant.Grantee.ID)) - if grant.Grantee.DisplayName != "" { - xml = append(xml, fmt.Sprintf("%s", grant.Grantee.DisplayName)) + if isObs { + xml = append(xml, "") + } + if grant.Grantee.ID != "" { + granteeID := XmlTranscoding(grant.Grantee.ID) + xml = append(xml, fmt.Sprintf("%s", granteeID)) } + if !isObs && grant.Grantee.DisplayName != "" { + granteeDisplayName := XmlTranscoding(grant.Grantee.DisplayName) + xml = append(xml, fmt.Sprintf("%s", granteeDisplayName)) + } + xml = append(xml, "") } else { - xml = append(xml, fmt.Sprintf("%s", grant.Grantee.URI)) + if !isObs { + if grant.Grantee.URI == GroupAllUsers || grant.Grantee.URI == GroupAuthenticatedUsers { + xml = append(xml, fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/global/", grant.Grantee.URI)) + } else if grant.Grantee.URI == GroupLogDelivery { + xml = append(xml, fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/s3/", grant.Grantee.URI)) + } else { + xml = append(xml, fmt.Sprintf("%s", grant.Grantee.URI)) + } + xml = append(xml, "") + } else if grant.Grantee.URI == GroupAllUsers { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("Everyone")) + xml = append(xml, "") + } else { + return strings.Join(xml, "") + } + } + + xml = append(xml, fmt.Sprintf("%s", grant.Permission)) + if isObs && isBucket { + xml = append(xml, fmt.Sprintf("%t", grant.Delivered)) } - xml = append(xml, fmt.Sprintf("%s", grant.Permission)) + xml = append(xml, fmt.Sprintf("")) return strings.Join(xml, "") } -func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool) (data string, md5 string) { +func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool, isObs bool) (data string, md5 string) { grantsLength := len(input.TargetGrants) xml := make([]string, 0, 8+grantsLength) xml = append(xml, "") - if input.TargetBucket != "" || input.TargetPrefix != "" { + if isObs && input.Agency != "" { + agency := XmlTranscoding(input.Agency) + xml = append(xml, fmt.Sprintf("%s", agency)) + } + if input.TargetBucket != "" || input.TargetPrefix != "" || grantsLength > 0 { xml = append(xml, "") - xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) - xml = append(xml, fmt.Sprintf("%s", input.TargetPrefix)) - + if input.TargetBucket != "" { + xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) + } + if input.TargetPrefix != "" { + targetPrefix := XmlTranscoding(input.TargetPrefix) + xml = append(xml, fmt.Sprintf("%s", targetPrefix)) + } if grantsLength > 0 { xml = append(xml, "") for _, grant := range input.TargetGrants { - xml = append(xml, convertGrantToXml(grant)) + xml = append(xml, convertGrantToXml(grant, isObs, false)) } xml = append(xml, "") } @@ -82,15 +157,44 @@ func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool) (data return } -func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool) (data string, md5 string) { +func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { xml := make([]string, 0, 4+len(input.Grants)) - xml = append(xml, fmt.Sprintf("%s", input.Owner.ID)) - if input.Owner.DisplayName != "" { - xml = append(xml, fmt.Sprintf("%s", input.Owner.DisplayName)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + if isObs && input.Delivered != "" { + objectDelivered := XmlTranscoding(input.Delivered) + xml = append(xml, fmt.Sprintf("%s", objectDelivered)) + } else { + xml = append(xml, "") + } + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXml(grant, isObs, false)) } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertBucketAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + xml = append(xml, "") + for _, grant := range input.Grants { - xml = append(xml, convertGrantToXml(grant)) + xml = append(xml, convertGrantToXml(grant, isObs, true)) } xml = append(xml, "") data = strings.Join(xml, "") @@ -103,7 +207,8 @@ func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool) (data string, md func convertConditionToXml(condition Condition) string { xml := make([]string, 0, 2) if condition.KeyPrefixEquals != "" { - xml = append(xml, fmt.Sprintf("%s", condition.KeyPrefixEquals)) + keyPrefixEquals := XmlTranscoding(condition.KeyPrefixEquals) + xml = append(xml, fmt.Sprintf("%s", keyPrefixEquals)) } if condition.HttpErrorCodeReturnedEquals != "" { xml = append(xml, fmt.Sprintf("%s", condition.HttpErrorCodeReturnedEquals)) @@ -126,9 +231,13 @@ func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd } xml = append(xml, "") } else { - xml = append(xml, fmt.Sprintf("%s", input.IndexDocument.Suffix)) + if input.IndexDocument.Suffix != "" { + indexDocumentSuffix := XmlTranscoding(input.IndexDocument.Suffix) + xml = append(xml, fmt.Sprintf("%s", indexDocumentSuffix)) + } if input.ErrorDocument.Key != "" { - xml = append(xml, fmt.Sprintf("%s", input.ErrorDocument.Key)) + errorDocumentKey := XmlTranscoding(input.ErrorDocument.Key) + xml = append(xml, fmt.Sprintf("%s", errorDocumentKey)) } if routingRuleLength > 0 { xml = append(xml, "") @@ -142,11 +251,13 @@ func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HostName)) } if routingRule.Redirect.ReplaceKeyPrefixWith != "" { - xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.ReplaceKeyPrefixWith)) + replaceKeyPrefixWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyPrefixWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyPrefixWith)) } if routingRule.Redirect.ReplaceKeyWith != "" { - xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.ReplaceKeyWith)) + replaceKeyWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyWith)) } if routingRule.Redirect.HttpRedirectCode != "" { xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HttpRedirectCode)) @@ -170,7 +281,7 @@ func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd return } -func convertTransitionsToXml(transitions []Transition) string { +func convertTransitionsToXml(transitions []Transition, isObs bool) string { if length := len(transitions); length > 0 { xml := make([]string, 0, length) for _, transition := range transitions { @@ -181,7 +292,17 @@ func convertTransitionsToXml(transitions []Transition) string { temp = fmt.Sprintf("%s", transition.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) } if temp != "" { - xml = append(xml, fmt.Sprintf("%s%s", temp, transition.StorageClass)) + if !isObs { + storageClass := string(transition.StorageClass) + if transition.StorageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if transition.StorageClass == "COLD" { + storageClass = "GLACIER" + } + xml = append(xml, fmt.Sprintf("%s%s", temp, storageClass)) + } else { + xml = append(xml, fmt.Sprintf("%s%s", temp, transition.StorageClass)) + } } } return strings.Join(xml, "") @@ -197,14 +318,22 @@ func convertExpirationToXml(expiration Expiration) string { } return "" } -func convertNoncurrentVersionTransitionsToXml(noncurrentVersionTransitions []NoncurrentVersionTransition) string { +func convertNoncurrentVersionTransitionsToXml(noncurrentVersionTransitions []NoncurrentVersionTransition, isObs bool) string { if length := len(noncurrentVersionTransitions); length > 0 { xml := make([]string, 0, length) for _, noncurrentVersionTransition := range noncurrentVersionTransitions { if noncurrentVersionTransition.NoncurrentDays > 0 { + storageClass := string(noncurrentVersionTransition.StorageClass) + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } xml = append(xml, fmt.Sprintf("%d"+ "%s", - noncurrentVersionTransition.NoncurrentDays, noncurrentVersionTransition.StorageClass)) + noncurrentVersionTransition.NoncurrentDays, storageClass)) } } return strings.Join(xml, "") @@ -218,23 +347,25 @@ func convertNoncurrentVersionExpirationToXml(noncurrentVersionExpiration Noncurr return "" } -func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, returnMd5 bool) (data string, md5 string) { +func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, returnMd5 bool, isObs bool) (data string, md5 string) { xml := make([]string, 0, 2+len(input.LifecycleRules)*9) xml = append(xml, "") for _, lifecyleRule := range input.LifecycleRules { xml = append(xml, "") if lifecyleRule.ID != "" { - xml = append(xml, fmt.Sprintf("%s", lifecyleRule.ID)) + lifecyleRuleID := XmlTranscoding(lifecyleRule.ID) + xml = append(xml, fmt.Sprintf("%s", lifecyleRuleID)) } - xml = append(xml, fmt.Sprintf("%s", lifecyleRule.Prefix)) + lifecyleRulePrefix := XmlTranscoding(lifecyleRule.Prefix) + xml = append(xml, fmt.Sprintf("%s", lifecyleRulePrefix)) xml = append(xml, fmt.Sprintf("%s", lifecyleRule.Status)) - if ret := convertTransitionsToXml(lifecyleRule.Transitions); ret != "" { + if ret := convertTransitionsToXml(lifecyleRule.Transitions, isObs); ret != "" { xml = append(xml, ret) } if ret := convertExpirationToXml(lifecyleRule.Expiration); ret != "" { xml = append(xml, ret) } - if ret := convertNoncurrentVersionTransitionsToXml(lifecyleRule.NoncurrentVersionTransitions); ret != "" { + if ret := convertNoncurrentVersionTransitionsToXml(lifecyleRule.NoncurrentVersionTransitions, isObs); ret != "" { xml = append(xml, ret) } if ret := convertNoncurrentVersionExpirationToXml(lifecyleRule.NoncurrentVersionExpiration); ret != "" { @@ -250,52 +381,85 @@ func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, return return } -func converntFilterRulesToXml(filterRules []FilterRule) string { +func converntFilterRulesToXml(filterRules []FilterRule, isObs bool) string { if length := len(filterRules); length > 0 { xml := make([]string, 0, length*4) for _, filterRule := range filterRules { xml = append(xml, "") if filterRule.Name != "" { - xml = append(xml, fmt.Sprintf("%s", filterRule.Name)) + filterRuleName := XmlTranscoding(filterRule.Name) + xml = append(xml, fmt.Sprintf("%s", filterRuleName)) } if filterRule.Value != "" { - xml = append(xml, fmt.Sprintf("%s", filterRule.Value)) + filterRuleValue := XmlTranscoding(filterRule.Value) + xml = append(xml, fmt.Sprintf("%s", filterRuleValue)) } xml = append(xml, "") } - return fmt.Sprintf("%s", strings.Join(xml, "")) + if !isObs { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } else { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } } return "" } -func converntEventsToXml(events []string) string { +func converntEventsToXml(events []EventType, isObs bool) string { if length := len(events); length > 0 { xml := make([]string, 0, length) - for _, event := range events { - xml = append(xml, fmt.Sprintf("%s", event)) + if !isObs { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s%s", "s3:", event)) + } + } else { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s", event)) + } } return strings.Join(xml, "") } return "" } -func ConvertNotificationToXml(input BucketNotification, returnMd5 bool) (data string, md5 string) { +func converntConfigureToXml(topicConfiguration TopicConfiguration, xmlElem string, isObs bool) string { + xml := make([]string, 0, 6) + xml = append(xml, xmlElem) + if topicConfiguration.ID != "" { + topicConfigurationID := XmlTranscoding(topicConfiguration.ID) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationID)) + } + topicConfigurationTopic := XmlTranscoding(topicConfiguration.Topic) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationTopic)) + + if ret := converntEventsToXml(topicConfiguration.Events, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := converntFilterRulesToXml(topicConfiguration.FilterRules, isObs); ret != "" { + xml = append(xml, ret) + } + tempElem := xmlElem[0:1] + "/" + xmlElem[1:] + xml = append(xml, tempElem) + return strings.Join(xml, "") +} + +func ConverntObsRestoreToXml(restoreObjectInput RestoreObjectInput) string { + xml := make([]string, 0, 2) + xml = append(xml, fmt.Sprintf("%d", restoreObjectInput.Days)) + if restoreObjectInput.Tier != "Bulk" { + xml = append(xml, fmt.Sprintf("%s", restoreObjectInput.Tier)) + } + xml = append(xml, fmt.Sprintf("")) + data := strings.Join(xml, "") + return data +} + +func ConvertNotificationToXml(input BucketNotification, returnMd5 bool, isObs bool) (data string, md5 string) { xml := make([]string, 0, 2+len(input.TopicConfigurations)*6) xml = append(xml, "") for _, topicConfiguration := range input.TopicConfigurations { - xml = append(xml, "") - if topicConfiguration.ID != "" { - xml = append(xml, fmt.Sprintf("%s", topicConfiguration.ID)) - } - xml = append(xml, fmt.Sprintf("%s", topicConfiguration.Topic)) - - if ret := converntEventsToXml(topicConfiguration.Events); ret != "" { - xml = append(xml, ret) - } - if ret := converntFilterRulesToXml(topicConfiguration.FilterRules); ret != "" { - xml = append(xml, ret) - } - xml = append(xml, "") + ret := converntConfigureToXml(topicConfiguration, "", isObs) + xml = append(xml, ret) } xml = append(xml, "") data = strings.Join(xml, "") @@ -333,6 +497,8 @@ func parseSseHeader(responseHeaders map[string][]string) (sseHeader ISseHeader) sseKmsHeader := SseKmsHeader{Encryption: ret[0]} if ret, ok = responseHeaders[HEADER_SSEKMS_KEY]; ok { sseKmsHeader.Key = ret[0] + } else if ret, ok = responseHeaders[HEADER_SSEKMS_ENCRYPT_KEY_OBS]; ok { + sseKmsHeader.Key = ret[0] } sseHeader = sseKmsHeader } @@ -352,7 +518,12 @@ func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { if ret, ok := output.ResponseHeaders[HEADER_RESTORE]; ok { output.Restore = ret[0] } - + if ret, ok := output.ResponseHeaders[HEADER_OBJECT_TYPE]; ok { + output.Restore = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_NEXT_APPEND_POSITION]; ok { + output.Restore = ret[0] + } if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { output.StorageClass = ParseStringToStorageClassType(ret[0]) } @@ -420,7 +591,6 @@ func ParsePutObjectOutput(output *PutObjectOutput) { if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { output.StorageClass = ParseStringToStorageClassType(ret[0]) } - if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { output.ETag = ret[0] } @@ -451,9 +621,16 @@ func ParseCopyPartOutput(output *CopyPartOutput) { func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_VERSION_OBS]; ok { + output.Version = ret[0] } if ret, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { output.Location = ret[0] + } else if ret, ok := output.ResponseHeaders[HEADER_BUCKET_LOCATION_OBS]; ok { + output.Location = ret[0] } if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { output.AllowOrigin = ret[0] @@ -470,8 +647,52 @@ func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { output.ExposeHeader = ret[0] } + if ret, ok := output.ResponseHeaders[HEADER_EPID_HEADERS]; ok { + output.Epid = ret[0] + } } +func ParseSetObjectMetadataOutput(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_METADATA_DIRECTIVE]; ok { + output.MetadataDirective = MetadataDirectiveType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } +} func ParseDeleteObjectOutput(output *DeleteObjectOutput) { if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { output.VersionId = versionId[0] @@ -526,7 +747,7 @@ func ConvertRequestToIoReader(req interface{}) (io.Reader, error) { return nil, err } -func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool) (err error) { +func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool, isObs bool) (err error) { readCloser, ok := baseModel.(IReadCloser) if !ok { defer resp.Body.Close() @@ -540,7 +761,7 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu } else { s := reflect.TypeOf(baseModel).Elem() for i := 0; i < s.NumField(); i++ { - if s.Field(i).Tag == "body" { + if s.Field(i).Tag == "json:\"body\"" { reflect.ValueOf(baseModel).Elem().FieldByName(s.Field(i).Name).SetString(string(body)) break } @@ -560,9 +781,12 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu return } -func ParseResponseToObsError(resp *http.Response) error { +func ParseResponseToObsError(resp *http.Response, isObs bool) error { obsError := ObsError{} - ParseResponseToBaseModel(resp, &obsError, true) + respError := ParseResponseToBaseModel(resp, &obsError, true, isObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } obsError.Status = resp.Status return obsError } diff --git a/openstack/obs/error.go b/openstack/obs/error.go index c79087ec0..eff74e611 100644 --- a/openstack/obs/error.go +++ b/openstack/obs/error.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( diff --git a/openstack/obs/http.go b/openstack/obs/http.go index 9b3a51077..de73e4912 100644 --- a/openstack/obs/http.go +++ b/openstack/obs/http.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -13,7 +25,7 @@ import ( "time" ) -func prepareHeaders(headers map[string][]string, meta bool) map[string][]string { +func prepareHeaders(headers map[string][]string, meta bool, isObs bool) map[string][]string { _headers := make(map[string][]string, len(headers)) if headers != nil { for key, value := range headers { @@ -22,11 +34,15 @@ func prepareHeaders(headers map[string][]string, meta bool) map[string][]string continue } _key := strings.ToLower(key) - if _, ok := allowed_request_http_header_metadata_names[_key]; !ok && !strings.HasPrefix(key, HEADER_PREFIX) { + if _, ok := allowed_request_http_header_metadata_names[_key]; !ok && !strings.HasPrefix(key, HEADER_PREFIX) && !strings.HasPrefix(key, HEADER_PREFIX_OBS) { if !meta { continue } - _key = HEADER_PREFIX_META + _key + if !isObs { + _key = HEADER_PREFIX_META + _key + } else { + _key = HEADER_PREFIX_META_OBS + _key + } } else { _key = key } @@ -41,14 +57,14 @@ func (obsClient ObsClient) doActionWithoutBucket(action, method string, input IS } func (obsClient ObsClient) doActionWithBucketV2(action, method, bucketName string, input ISerializable, output IBaseModel) error { - if strings.TrimSpace(bucketName) == "" { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { return errors.New("Bucket is empty") } return obsClient.doAction(action, method, bucketName, "", input, output, false, true) } func (obsClient ObsClient) doActionWithBucket(action, method, bucketName string, input ISerializable, output IBaseModel) error { - if strings.TrimSpace(bucketName) == "" { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { return errors.New("Bucket is empty") } return obsClient.doAction(action, method, bucketName, "", input, output, true, true) @@ -63,8 +79,8 @@ func (obsClient ObsClient) doActionWithBucketAndKeyUnRepeatable(action, method, } func (obsClient ObsClient) _doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, repeatable bool) error { - if strings.TrimSpace(bucketName) == "" { - return errors.New("Key is empty") + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") } if strings.TrimSpace(objectKey) == "" { return errors.New("Key is empty") @@ -79,8 +95,10 @@ func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string doLog(LEVEL_INFO, "Enter method %s...", action) start := GetCurrentTimestamp() - params, headers, data := input.trans() - + params, headers, data, err := input.trans(obsClient.conf.signature == SignatureObs) + if err != nil { + return err + } if params == nil { params = make(map[string]string) } @@ -106,46 +124,49 @@ func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string respError = errors.New("Unexpect http method error") } if respError == nil && output != nil { - respError = ParseResponseToBaseModel(resp, output, xmlResult) + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) if respError != nil { doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) } } else { doLog(LEVEL_WARN, "Do http request with error: %v", respError) } - doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } return respError } func (obsClient ObsClient) doHttpGet(bucketName, objectKey string, params map[string]string, headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { - return obsClient.doHttp(HTTP_GET, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable) + return obsClient.doHttp(HTTP_GET, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) } func (obsClient ObsClient) doHttpHead(bucketName, objectKey string, params map[string]string, headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { - return obsClient.doHttp(HTTP_HEAD, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable) + return obsClient.doHttp(HTTP_HEAD, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) } func (obsClient ObsClient) doHttpOptions(bucketName, objectKey string, params map[string]string, headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { - return obsClient.doHttp(HTTP_OPTIONS, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable) + return obsClient.doHttp(HTTP_OPTIONS, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) } func (obsClient ObsClient) doHttpDelete(bucketName, objectKey string, params map[string]string, headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { - return obsClient.doHttp(HTTP_DELETE, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable) + return obsClient.doHttp(HTTP_DELETE, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) } func (obsClient ObsClient) doHttpPut(bucketName, objectKey string, params map[string]string, headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { - return obsClient.doHttp(HTTP_PUT, bucketName, objectKey, params, prepareHeaders(headers, true), data, repeatable) + return obsClient.doHttp(HTTP_PUT, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) } func (obsClient ObsClient) doHttpPost(bucketName, objectKey string, params map[string]string, headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { - return obsClient.doHttp(HTTP_POST, bucketName, objectKey, params, prepareHeaders(headers, true), data, repeatable) + return obsClient.doHttp(HTTP_POST, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) } func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader, output IBaseModel, xmlResult bool) (respError error) { @@ -153,7 +174,9 @@ func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl if err != nil { return err } - + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } var resp *http.Response doLog(LEVEL_INFO, "Do %s with signedUrl %s...", action, signedUrl) @@ -178,7 +201,9 @@ func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT} start := GetCurrentTimestamp() resp, err = obsClient.httpClient.Do(req) - doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } var msg interface{} if err != nil { @@ -187,12 +212,12 @@ func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl } else { doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) if resp.StatusCode >= 300 { - respError = ParseResponseToObsError(resp) + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) msg = resp.Status resp = nil } else { if output != nil { - respError = ParseResponseToBaseModel(resp, output, xmlResult) + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) } if respError != nil { doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) @@ -203,7 +228,10 @@ func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl if msg != nil { doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) } - doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } return } @@ -213,13 +241,12 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m bucketName = strings.TrimSpace(bucketName) - objectKey = strings.TrimSpace(objectKey) - method = strings.ToUpper(method) var redirectUrl string var requestUrl string maxRetryCount := obsClient.conf.maxRetryCount + maxRedirectCount := obsClient.conf.maxRedirectCount var _data io.Reader if data != nil { @@ -239,15 +266,21 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } } - for i := 0; i <= maxRetryCount; i++ { + var lastRequest *http.Request + redirectFlag := false + for i, redirectCount := 0, 0; i <= maxRetryCount; i++ { if redirectUrl != "" { - parsedRedirectUrl, err := url.Parse(redirectUrl) - if err != nil { - return nil, err - } - requestUrl, _ = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectUrl.Host) - if parsedRequestUrl, _ := url.Parse(requestUrl); parsedRequestUrl.RawQuery != "" && parsedRedirectUrl.RawQuery == "" { - redirectUrl += "?" + parsedRequestUrl.RawQuery + if !redirectFlag { + parsedRedirectUrl, err := url.Parse(redirectUrl) + if err != nil { + return nil, err + } + requestUrl, _ = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectUrl.Host) + if parsedRequestUrl, err := url.Parse(requestUrl); err != nil { + return nil, err + } else if parsedRequestUrl.RawQuery != "" && parsedRedirectUrl.RawQuery == "" { + redirectUrl += "?" + parsedRequestUrl.RawQuery + } } requestUrl = redirectUrl } else { @@ -259,6 +292,9 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } req, err := http.NewRequest(method, requestUrl, _data) + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } if err != nil { return nil, err } @@ -283,33 +319,51 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } } + lastRequest = req + req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT} + if lastRequest != nil { + req.Host = lastRequest.Host + req.ContentLength = lastRequest.ContentLength + } + start := GetCurrentTimestamp() resp, err = obsClient.httpClient.Do(req) - doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } var msg interface{} if err != nil { msg = err respError = err resp = nil + if !repeatable { + break + } } else { doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) if resp.StatusCode < 300 { break - } else if !repeatable || (resp.StatusCode >= 300 && resp.StatusCode < 500 && resp.StatusCode != 307) { - respError = ParseResponseToObsError(resp) + } else if !repeatable || (resp.StatusCode >= 400 && resp.StatusCode < 500) || resp.StatusCode == 304 { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) resp = nil break - } else if resp.StatusCode == 307 { - if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" { + } else if resp.StatusCode >= 300 && resp.StatusCode < 400 { + if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" && redirectCount < maxRedirectCount { redirectUrl = location doLog(LEVEL_WARN, "Redirect request to %s", redirectUrl) msg = resp.Status maxRetryCount++ + redirectCount++ + if resp.StatusCode == 302 && method == HTTP_GET { + redirectFlag = true + } else { + redirectFlag = false + } } else { - respError = ParseResponseToObsError(resp) + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) resp = nil break } @@ -318,14 +372,27 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } } if i != maxRetryCount { + if resp != nil { + _err := resp.Body.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close resp body with reason: %v", _err) + } + resp = nil + } if _, ok := headers[HEADER_AUTH_CAMEL]; ok { delete(headers, HEADER_AUTH_CAMEL) } doLog(LEVEL_WARN, "Failed to send request with reason:%v, will try again", msg) if r, ok := _data.(*strings.Reader); ok { - r.Seek(0, 0) + _, err := r.Seek(0, 0) + if err != nil { + return nil, err + } } else if r, ok := _data.(*bytes.Reader); ok { - r.Seek(0, 0) + _, err := r.Seek(0, 0) + if err != nil { + return nil, err + } } else if r, ok := _data.(*fileReaderWrapper); ok { fd, err := os.Open(r.filePath) if err != nil { @@ -337,15 +404,21 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m fileReaderWrapper.reader = fd fileReaderWrapper.totalCount = r.totalCount _data = fileReaderWrapper - fd.Seek(r.mark, 0) + _, err = fd.Seek(r.mark, 0) + if err != nil { + return nil, err + } } else if r, ok := _data.(*readerWrapper); ok { - r.seek(0, 0) + _, err := r.seek(0, 0) + if err != nil { + return nil, err + } } time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second))) } else { doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) if resp != nil { - respError = ParseResponseToObsError(resp) + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) resp = nil } } @@ -368,18 +441,38 @@ func getConnDelegate(conn net.Conn, socketTimeout int, finalTimeout int) *connDe } func (delegate *connDelegate) Read(b []byte) (n int, err error) { - delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + setReadDeadlineErr := delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + n, err = delegate.conn.Read(b) - delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + setReadDeadlineErr = delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } return n, err } func (delegate *connDelegate) Write(b []byte) (n int, err error) { - delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + setWriteDeadlineErr := delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + n, err = delegate.conn.Write(b) finalTimeout := time.Now().Add(delegate.finalTimeout) - delegate.SetWriteDeadline(finalTimeout) - delegate.SetReadDeadline(finalTimeout) + setWriteDeadlineErr = delegate.SetWriteDeadline(finalTimeout) + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + setReadDeadlineErr := delegate.SetReadDeadline(finalTimeout) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } return n, err } diff --git a/openstack/obs/log.go b/openstack/obs/log.go index ed0148251..f411180b5 100644 --- a/openstack/obs/log.go +++ b/openstack/obs/log.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -21,8 +33,6 @@ const ( LEVEL_DEBUG Level = 100 ) -const cacheCount = 50 - var logLevelMap = map[Level]string{ LEVEL_OFF: "[OFF]: ", LEVEL_ERROR: "[ERROR]: ", @@ -52,30 +62,44 @@ func getDefaultLogConf() logConfType { var logConf logConfType type loggerWrapper struct { - fullPath string - fd *os.File - queue []string - logger *log.Logger - index int - lock *sync.RWMutex + fullPath string + fd *os.File + ch chan string + wg sync.WaitGroup + queue []string + logger *log.Logger + index int + cacheCount int + closed bool } func (lw *loggerWrapper) doInit() { - lw.queue = make([]string, 0, cacheCount) + lw.queue = make([]string, 0, lw.cacheCount) lw.logger = log.New(lw.fd, "", 0) - lw.lock = new(sync.RWMutex) + lw.ch = make(chan string, lw.cacheCount) + lw.wg.Add(1) + go lw.doWrite() } func (lw *loggerWrapper) rotate() { stat, err := lw.fd.Stat() - if err == nil && stat.Size() >= logConf.maxLogSize { - lw.fd.Sync() + if err != nil { + lw.fd.Close() + panic(err) + } + if stat.Size() >= logConf.maxLogSize { + _err := lw.fd.Sync() + if _err != nil { + panic(err) + } lw.fd.Close() - if lw.index > logConf.backups { lw.index = 1 } - os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) + _err = os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) + if _err != nil { + panic(err) + } lw.index += 1 fd, err := os.OpenFile(lw.fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) @@ -92,40 +116,45 @@ func (lw *loggerWrapper) doFlush() { for _, m := range lw.queue { lw.logger.Println(m) } - lw.fd.Sync() + err := lw.fd.Sync() + if err != nil { + panic(err) + } } func (lw *loggerWrapper) doClose() { - lw.doFlush() - lw.fd.Close() - lw.queue = nil - lw.fd = nil - lw.logger = nil - lw.lock = nil - lw.fullPath = "" + lw.closed = true + close(lw.ch) + lw.wg.Wait() } -func (lw *loggerWrapper) Printf(format string, v ...interface{}) { - msg := fmt.Sprintf(format, v...) - if len(lw.queue) >= cacheCount { - lw.lock.Lock() - defer lw.lock.Unlock() - if len(lw.queue) >= cacheCount { +func (lw *loggerWrapper) doWrite() { + defer lw.wg.Done() + for { + msg, ok := <-lw.ch + if !ok { lw.doFlush() - lw.queue = make([]string, 0, cacheCount) - } else { - lw.queue = append(lw.queue, msg) + lw.fd.Close() + break + } + if len(lw.queue) >= lw.cacheCount { + lw.doFlush() + lw.queue = make([]string, 0, lw.cacheCount) } - } else { - lock.RLock() - defer lock.RUnlock() lw.queue = append(lw.queue, msg) } + +} + +func (lw *loggerWrapper) Printf(format string, v ...interface{}) { + if !lw.closed { + msg := fmt.Sprintf(format, v...) + lw.ch <- msg + } } var consoleLogger *log.Logger var fileLogger *loggerWrapper - var lock *sync.RWMutex = new(sync.RWMutex) func isDebugLogEnabled() bool { @@ -154,8 +183,15 @@ func reset() { } func InitLog(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool) error { + return InitLogWithCacheCnt(logFullPath, maxLogSize, backups, level, logToConsole, 50) +} + +func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool, cacheCnt int) error { lock.Lock() defer lock.Unlock() + if cacheCnt <= 0 { + cacheCnt = 50 + } reset() if fullPath := strings.TrimSpace(logFullPath); fullPath != "" { _fullPath, err := filepath.Abs(fullPath) @@ -178,22 +214,34 @@ func InitLog(logFullPath string, maxLogSize int64, backups int, level Level, log if err != nil { return err } - fileLogger = &loggerWrapper{fullPath: _fullPath, fd: fd, index: 1} if stat == nil { stat, err = os.Stat(_fullPath) + if err != nil { + fd.Close() + return err + } } + prefix := stat.Name() + "." + index := 1 walkFunc := func(path string, info os.FileInfo, err error) error { - if name := info.Name(); strings.HasPrefix(name, prefix) { - if i := StringToInt(name[len(prefix):], 0); i >= fileLogger.index { - fileLogger.index = i + 1 + if err == nil { + if name := info.Name(); strings.HasPrefix(name, prefix) { + if i := StringToInt(name[len(prefix):], 0); i >= index { + index = i + 1 + } } } - return nil + return err } - filepath.Walk(filepath.Dir(_fullPath), walkFunc) + if err = filepath.Walk(filepath.Dir(_fullPath), walkFunc); err != nil { + fd.Close() + return err + } + + fileLogger = &loggerWrapper{fullPath: _fullPath, fd: fd, index: index, cacheCount: cacheCnt, closed: false} fileLogger.doInit() } if maxLogSize > 0 { @@ -210,7 +258,7 @@ func InitLog(logFullPath string, maxLogSize int64, backups int, level Level, log } func CloseLog() { - if fileLogger != nil || consoleLogger != nil { + if logEnabled() { lock.Lock() defer lock.Unlock() reset() @@ -218,21 +266,18 @@ func CloseLog() { } func SyncLog() { - if fileLogger != nil { - lock.Lock() - defer lock.Unlock() - fileLogger.doFlush() - } } func logEnabled() bool { return consoleLogger != nil || fileLogger != nil } +func DoLog(level Level, format string, v ...interface{}) { + doLog(level, format, v...) +} + func doLog(level Level, format string, v ...interface{}) { if logEnabled() && logConf.level <= level { - lock.RLock() - defer lock.RUnlock() msg := fmt.Sprintf(format, v...) if _, file, line, ok := runtime.Caller(1); ok { index := strings.LastIndex(file, "/") @@ -251,25 +296,3 @@ func doLog(level Level, format string, v ...interface{}) { } } } - -func LOG(level Level, format string, v ...interface{}) { - if logEnabled() && logConf.level <= level { - lock.RLock() - defer lock.RUnlock() - msg := fmt.Sprintf(format, v...) - if _, file, line, ok := runtime.Caller(1); ok { - index := strings.LastIndex(file, "/") - if index >= 0 { - file = file[index+1:] - } - msg = fmt.Sprintf("%s:%d|%s", file, line, msg) - } - prefix := logLevelMap[level] - if consoleLogger != nil { - consoleLogger.Printf("%s%s", prefix, msg) - } - if fileLogger != nil { - fileLogger.Printf("%s%s", prefix, msg) - } - } -} diff --git a/openstack/obs/model.go b/openstack/obs/model.go index fb952bec4..04596657b 100644 --- a/openstack/obs/model.go +++ b/openstack/obs/model.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -17,6 +29,7 @@ type Bucket struct { XMLName xml.Name `xml:"Bucket"` Name string `xml:"Name"` CreationDate time.Time `xml:"CreationDate"` + Location string `xml:"Location"` } type Owner struct { @@ -32,6 +45,7 @@ type Initiator struct { } type ListBucketsInput struct { + QueryLocation bool } type ListBucketsOutput struct { @@ -41,16 +55,29 @@ type ListBucketsOutput struct { Buckets []Bucket `xml:"Buckets>Bucket"` } +type bucketLocationObs struct { + XMLName xml.Name `xml:"Location"` + Location string `xml:",chardata"` +} + type BucketLocation struct { XMLName xml.Name `xml:"CreateBucketConfiguration"` - Location string `xml:"LocationConstraint"` + Location string `xml:"LocationConstraint,omitempty"` } type CreateBucketInput struct { BucketLocation - Bucket string `xml:"-"` - ACL AclType `xml:"-"` - StorageClass StorageClassType `xml:"-"` + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + StorageClass StorageClassType `xml:"-"` + GrantReadId string `xml:"-"` + GrantWriteId string `xml:"-"` + GrantReadAcpId string `xml:"-"` + GrantWriteAcpId string `xml:"-"` + GrantFullControlId string `xml:"-"` + GrantReadDeliveredId string `xml:"-"` + GrantFullControlDeliveredId string `xml:"-"` + Epid string `xml:"-"` } type BucketStoragePolicy struct { @@ -63,11 +90,25 @@ type SetBucketStoragePolicyInput struct { BucketStoragePolicy } -type GetBucketStoragePolicyOutput struct { +type getBucketStoragePolicyOutputS3 struct { BaseModel BucketStoragePolicy } +type GetBucketStoragePolicyOutput struct { + BaseModel + StorageClass string +} + +type bucketStoragePolicyObs struct { + XMLName xml.Name `xml:"StorageClass"` + StorageClass string `xml:",chardata"` +} +type getBucketStoragePolicyOutputObs struct { + BaseModel + bucketStoragePolicyObs +} + type ListObjsInput struct { Prefix string MaxKeys int @@ -206,10 +247,18 @@ type GetBucketStorageInfoOutput struct { ObjectNumber int `xml:"ObjectNumber"` } -type GetBucketLocationOutput struct { +type getBucketLocationOutputS3 struct { BaseModel BucketLocation } +type getBucketLocationOutputObs struct { + BaseModel + bucketLocationObs +} +type GetBucketLocationOutput struct { + BaseModel + Location string `xml:"-"` +} type Grantee struct { XMLName xml.Name `xml:"Grantee"` @@ -219,16 +268,38 @@ type Grantee struct { URI GroupUriType `xml:"URI,omitempty"` } +type granteeObs struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + Canned string `xml:"Canned,omitempty"` +} + type Grant struct { XMLName xml.Name `xml:"Grant"` Grantee Grantee `xml:"Grantee"` Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} +type grantObs struct { + XMLName xml.Name `xml:"Grant"` + Grantee granteeObs `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` } type AccessControlPolicy struct { - XMLName xml.Name `xml:"AccessControlPolicy"` - Owner Owner `xml:"Owner"` - Grants []Grant `xml:"AccessControlList>Grant"` + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []Grant `xml:"AccessControlList>Grant"` + Delivered string `xml:"Delivered,omitempty"` +} + +type accessControlPolicyObs struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []grantObs `xml:"AccessControlList>Grant"` } type GetBucketAclOutput struct { @@ -236,6 +307,11 @@ type GetBucketAclOutput struct { AccessControlPolicy } +type getBucketAclOutputObs struct { + BaseModel + accessControlPolicyObs +} + type SetBucketAclInput struct { Bucket string `xml:"-"` ACL AclType `xml:"-"` @@ -249,7 +325,7 @@ type SetBucketPolicyInput struct { type GetBucketPolicyOutput struct { BaseModel - Policy string + Policy string `json:"body"` } type CorsRule struct { @@ -351,19 +427,52 @@ type GetBucketMetadataInput struct { RequestHeader string } +type SetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + +type SetObjectMetadataOutput struct { + BaseModel + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + type GetBucketMetadataOutput struct { BaseModel StorageClass StorageClassType Location string + Version string AllowOrigin string AllowMethod string AllowHeader string MaxAgeSeconds int ExposeHeader string + Epid string } type BucketLoggingStatus struct { XMLName xml.Name `xml:"BucketLoggingStatus"` + Agency string `xml:"Agency,omitempty"` TargetBucket string `xml:"LoggingEnabled>TargetBucket,omitempty"` TargetPrefix string `xml:"LoggingEnabled>TargetPrefix,omitempty"` TargetGrants []Grant `xml:"LoggingEnabled>TargetGrants>Grant,omitempty"` @@ -459,8 +568,8 @@ type TopicConfiguration struct { XMLName xml.Name `xml:"TopicConfiguration"` ID string `xml:"Id,omitempty"` Topic string `xml:"Topic"` - Events []string `xml:"Event"` - FilterRules []FilterRule `xml:"Filter>S3Key>FilterRule"` + Events []EventType `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>Object>FilterRule"` } type BucketNotification struct { @@ -473,6 +582,24 @@ type SetBucketNotificationInput struct { BucketNotification } +type topicConfigurationS3 struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []string `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>S3Key>FilterRule"` +} + +type bucketNotificationS3 struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []topicConfigurationS3 `xml:"TopicConfiguration"` +} + +type getBucketNotificationOutputS3 struct { + BaseModel + bucketNotificationS3 +} + type GetBucketNotificationOutput struct { BaseModel BucketNotification @@ -563,6 +690,7 @@ type ISseHeader interface { type SseKmsHeader struct { Encryption string Key string + isObs bool } type SseCHeader struct { @@ -586,6 +714,8 @@ type GetObjectMetadataOutput struct { WebsiteRedirectLocation string Expiration string Restore string + ObjectType string + NextAppendPosition string StorageClass StorageClassType ContentLength int64 ContentType string @@ -632,8 +762,13 @@ type ObjectOperationInput struct { Bucket string Key string ACL AclType + GrantReadId string + GrantReadAcpId string + GrantWriteAcpId string + GrantFullControlId string StorageClass StorageClassType WebsiteRedirectLocation string + Expires int64 SseHeader ISseHeader Metadata map[string]string } @@ -680,6 +815,7 @@ type CopyObjectInput struct { ContentType string Expires string MetadataDirective MetadataDirectiveType + SuccessActionRedirect string } type CopyObjectOutput struct { diff --git a/openstack/obs/temporary.go b/openstack/obs/temporary.go index 9e457f512..67893e846 100644 --- a/openstack/obs/temporary.go +++ b/openstack/obs/temporary.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -71,7 +83,11 @@ func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBased params[PARAM_DATE_AMZ_CAMEL] = longDate if obsClient.conf.securityProvider.securityToken != "" { - params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = obsClient.conf.securityProvider.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + } } matchAnyBucket := true diff --git a/openstack/obs/trait.go b/openstack/obs/trait.go index 49bfcc7ec..5ff3e3c63 100644 --- a/openstack/obs/trait.go +++ b/openstack/obs/trait.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -16,6 +28,24 @@ func (output *GetObjectOutput) setReadCloser(body io.ReadCloser) { output.Body = body } +func setHeaders(headers map[string][]string, header string, headerValue []string, isObs bool) { + if isObs { + header = HEADER_PREFIX_OBS + header + headers[header] = headerValue + } else { + header = HEADER_PREFIX + header + headers[header] = headerValue + } +} + +func setHeadersNext(headers map[string][]string, header string, headerNext string, headerValue []string, isObs bool) { + if isObs { + headers[header] = headerValue + } else { + headers[headerNext] = headerValue + } +} + type IBaseModel interface { setStatusCode(statusCode int) @@ -25,7 +55,7 @@ type IBaseModel interface { } type ISerializable interface { - trans() (map[string]string, map[string][]string, interface{}) + trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) } type DefaultSerializable struct { @@ -34,8 +64,8 @@ type DefaultSerializable struct { data interface{} } -func (input DefaultSerializable) trans() (map[string]string, map[string][]string, interface{}) { - return input.params, input.headers, input.data +func (input DefaultSerializable) trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) { + return input.params, input.headers, input.data, nil } var defaultSerializable = &DefaultSerializable{} @@ -44,9 +74,9 @@ func newSubResourceSerial(subResource SubResourceType) *DefaultSerializable { return &DefaultSerializable{map[string]string{string(subResource): ""}, nil, nil} } -func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}) { +func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(subResource): ""} - data, _ = ConvertRequestToIoReader(input) + data, err = ConvertRequestToIoReader(input) return } @@ -62,32 +92,93 @@ func (baseModel *BaseModel) setResponseHeaders(responseHeaders map[string][]stri baseModel.ResponseHeaders = responseHeaders } -func (input ListBucketsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListBucketsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if input.QueryLocation && !isObs { + setHeaders(headers, HEADER_LOCATION_AMZ, []string{"true"}, isObs) + } return } -func (input CreateBucketInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input CreateBucketInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) if acl := string(input.ACL); acl != "" { - headers[HEADER_ACL_AMZ] = []string{acl} + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) } - if storageClass := string(input.StorageClass); storageClass != "" { - headers[HEADER_STORAGE_CLASS] = []string{storageClass} + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } + setHeadersNext(headers, HEADER_STORAGE_CLASS_OBS, HEADER_STORAGE_CLASS, []string{storageClass}, isObs) + if epid := string(input.Epid); epid != "" { + setHeaders(headers, HEADER_EPID_HEADERS, []string{epid}, isObs) + } + } + if grantReadId := string(input.GrantReadId); grantReadId != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{grantReadId}, isObs) + } + if grantWriteId := string(input.GrantWriteId); grantWriteId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_OBS, []string{grantWriteId}, isObs) + } + if grantReadAcpId := string(input.GrantReadAcpId); grantReadAcpId != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{grantReadAcpId}, isObs) + } + if grantWriteAcpId := string(input.GrantWriteAcpId); grantWriteAcpId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{grantWriteAcpId}, isObs) + } + if grantFullControlId := string(input.GrantFullControlId); grantFullControlId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{grantFullControlId}, isObs) + } + if grantReadDeliveredId := string(input.GrantReadDeliveredId); grantReadDeliveredId != "" { + setHeaders(headers, HEADER_GRANT_READ_DELIVERED_OBS, []string{grantReadDeliveredId}, true) + } + if grantFullControlDeliveredId := string(input.GrantFullControlDeliveredId); grantFullControlDeliveredId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS, []string{grantFullControlDeliveredId}, true) } - if location := strings.TrimSpace(input.Location); location != "" { input.Location = location - data, _ = ConvertRequestToIoReader(input) + + xml := make([]string, 0, 3) + xml = append(xml, "") + if isObs { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } else { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } + xml = append(xml, "") + + data = strings.Join(xml, "") } return } -func (input SetBucketStoragePolicyInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - return trans(SubResourceStoragePolicy, input) +func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + xml := make([]string, 0, 1) + if !isObs { + storageClass := "STANDARD" + if input.StorageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if input.StorageClass == "COLD" { + storageClass = "GLACIER" + } + params = map[string]string{string(SubResourceStoragePolicy): ""} + xml = append(xml, fmt.Sprintf("%s", storageClass)) + } else { + if input.StorageClass != "WARM" && input.StorageClass != "COLD" { + input.StorageClass = StorageClassStandard + } + params = map[string]string{string(SubResourceStorageClass): ""} + xml = append(xml, fmt.Sprintf("%s", input.StorageClass)) + } + data = strings.Join(xml, "") + return } -func (input ListObjsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = make(map[string]string) if input.Prefix != "" { params["prefix"] = input.Prefix @@ -108,16 +199,22 @@ func (input ListObjsInput) trans() (params map[string]string, headers map[string return } -func (input ListObjectsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ListObjsInput.trans() +func (input ListObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } if input.Marker != "" { params["marker"] = input.Marker } return } -func (input ListVersionsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ListObjsInput.trans() +func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } params[string(SubResourceVersions)] = "" if input.KeyMarker != "" { params["key-marker"] = input.KeyMarker @@ -128,7 +225,7 @@ func (input ListVersionsInput) trans() (params map[string]string, headers map[st return } -func (input ListMultipartUploadsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceUploads): ""} if input.Prefix != "" { params["prefix"] = input.Prefix @@ -148,46 +245,49 @@ func (input ListMultipartUploadsInput) trans() (params map[string]string, header return } -func (input SetBucketQuotaInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketQuotaInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { return trans(SubResourceQuota, input) } -func (input SetBucketAclInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceAcl): ""} headers = make(map[string][]string) if acl := string(input.ACL); acl != "" { - headers[HEADER_ACL_AMZ] = []string{acl} + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) } else { - data, _ = ConvertAclToXml(input.AccessControlPolicy, false) + data, _ = convertBucketAclToXml(input.AccessControlPolicy, false, isObs) } return } -func (input SetBucketPolicyInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourcePolicy): ""} data = strings.NewReader(input.Policy) return } -func (input SetBucketCorsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceCors): ""} - data, md5, _ := ConvertRequestToIoReaderV2(input) + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetBucketVersioningInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketVersioningInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { return trans(SubResourceVersioning, input) } -func (input SetBucketWebsiteConfigurationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketWebsiteConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceWebsite): ""} data, _ = ConvertWebsiteConfigurationToXml(input.BucketWebsiteConfiguration, false) return } -func (input GetBucketMetadataInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) if origin := strings.TrimSpace(input.Origin); origin != "" { headers[HEADER_ORIGIN_CAMEL] = []string{origin} @@ -198,33 +298,36 @@ func (input GetBucketMetadataInput) trans() (params map[string]string, headers m return } -func (input SetBucketLoggingConfigurationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceLogging): ""} - data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false) + data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false, isObs) return } -func (input SetBucketLifecycleConfigurationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceLifecycle): ""} - data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true) + data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true, isObs) headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetBucketTaggingInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceTagging): ""} - data, md5, _ := ConvertRequestToIoReaderV2(input) + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetBucketNotificationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketNotificationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceNotification): ""} - data, _ = ConvertNotificationToXml(input.BucketNotification, false) + data, _ = ConvertNotificationToXml(input.BucketNotification, false, isObs) return } -func (input DeleteObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = make(map[string]string) if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -232,28 +335,31 @@ func (input DeleteObjectInput) trans() (params map[string]string, headers map[st return } -func (input DeleteObjectsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceDelete): ""} - data, md5, _ := ConvertRequestToIoReaderV2(input) + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetObjectAclInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceAcl): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId } headers = make(map[string][]string) if acl := string(input.ACL); acl != "" { - headers[HEADER_ACL_AMZ] = []string{acl} + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) } else { - data, _ = ConvertAclToXml(input.AccessControlPolicy, false) + data, _ = ConvertAclToXml(input.AccessControlPolicy, false, isObs) } return } -func (input GetObjectAclInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceAcl): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -261,12 +367,16 @@ func (input GetObjectAclInput) trans() (params map[string]string, headers map[st return } -func (input RestoreObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input RestoreObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceRestore): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId } - data, _ = ConvertRequestToIoReader(input) + if !isObs { + data, err = ConvertRequestToIoReader(input) + } else { + data = ConverntObsRestoreToXml(input) + } return } @@ -274,7 +384,11 @@ func (header SseKmsHeader) GetEncryption() string { if header.Encryption != "" { return header.Encryption } - return DEFAULT_SSE_KMS_ENCRYPTION + if !header.isObs { + return DEFAULT_SSE_KMS_ENCRYPTION + } else { + return DEFAULT_SSE_KMS_ENCRYPTION_OBS + } } func (header SseKmsHeader) GetKey() string { @@ -303,20 +417,21 @@ func (header SseCHeader) GetKeyMD5() string { return "" } -func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bool) { +func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bool, isObs bool) { if sseHeader != nil { if sseCHeader, ok := sseHeader.(SseCHeader); ok { - headers[HEADER_SSEC_ENCRYPTION_AMZ] = []string{sseCHeader.GetEncryption()} - headers[HEADER_SSEC_KEY_AMZ] = []string{sseCHeader.GetKey()} - headers[HEADER_SSEC_KEY_MD5_AMZ] = []string{sseCHeader.GetKeyMD5()} + setHeaders(headers, HEADER_SSEC_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY_MD5, []string{sseCHeader.GetEncryption()}, isObs) } else if sseKmsHeader, ok := sseHeader.(SseKmsHeader); !sseCOnly && ok { - headers[HEADER_SSEKMS_ENCRYPTION_AMZ] = []string{sseKmsHeader.GetEncryption()} - headers[HEADER_SSEKMS_KEY_AMZ] = []string{sseKmsHeader.GetKey()} + sseKmsHeader.isObs = isObs + setHeaders(headers, HEADER_SSEKMS_ENCRYPTION, []string{sseKmsHeader.GetEncryption()}, isObs) + setHeadersNext(headers, HEADER_SSEKMS_KEY_OBS, HEADER_SSEKMS_KEY_AMZ, []string{sseKmsHeader.GetKey()}, isObs) } } } -func (input GetObjectMetadataInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = make(map[string]string) if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -330,12 +445,69 @@ func (input GetObjectMetadataInput) trans() (params map[string]string, headers m if input.RequestHeader != "" { headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{input.RequestHeader} } - setSseHeader(headers, input.SseHeader, true) + setSseHeader(headers, input.SseHeader, true, isObs) + return +} + +func (input SetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + params = map[string]string{string(SubResourceMetadata): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(input.MetadataDirective)}, isObs) + } else { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(ReplaceNew)}, isObs) + } + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL_CAMEL] = []string{input.CacheControl} + } + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION_CAMEL] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING_CAMEL] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE_CAMEL] = []string{input.ContentLanguage} + } + + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } + if input.Expires != "" { + headers[HEADER_EXPIRES_CAMEL] = []string{input.Expires} + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + } + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } return } -func (input GetObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.GetObjectMetadataInput.trans() +func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.GetObjectMetadataInput.trans(isObs) + if err != nil { + return + } if input.ResponseCacheControl != "" { params[PARAM_RESPONSE_CACHE_CONTROL] = input.ResponseCacheControl } @@ -376,33 +548,56 @@ func (input GetObjectInput) trans() (params map[string]string, headers map[strin return } -func (input ObjectOperationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) params = make(map[string]string) if acl := string(input.ACL); acl != "" { - headers[HEADER_ACL_AMZ] = []string{acl} + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + if GrantReadId := string(input.GrantReadId); GrantReadId != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{GrantReadId}, true) + } + if GrantReadAcpId := string(input.GrantReadAcpId); GrantReadAcpId != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{GrantReadAcpId}, true) + } + if GrantWriteAcpId := string(input.GrantWriteAcpId); GrantWriteAcpId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{GrantWriteAcpId}, true) + } + if GrantFullControlId := string(input.GrantFullControlId); GrantFullControlId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{GrantFullControlId}, true) } if storageClass := string(input.StorageClass); storageClass != "" { - headers[HEADER_STORAGE_CLASS2_AMZ] = []string{storageClass} + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) } if input.WebsiteRedirectLocation != "" { - headers[HEADER_WEBSITE_REDIRECT_LOCATION_AMZ] = []string{input.WebsiteRedirectLocation} + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + + } + setSseHeader(headers, input.SseHeader, false, isObs) + if input.Expires != 0 { + setHeaders(headers, HEADER_EXPIRES, []string{Int64ToString(input.Expires)}, true) } - setSseHeader(headers, input.SseHeader, false) if input.Metadata != nil { for key, value := range input.Metadata { key = strings.TrimSpace(key) - if !strings.HasPrefix(key, HEADER_PREFIX_META) { - key = HEADER_PREFIX_META + key - } - headers[key] = []string{value} + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) } } return } -func (input PutObjectBasicInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ObjectOperationInput.trans() +func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } if input.ContentMD5 != "" { headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} @@ -418,27 +613,33 @@ func (input PutObjectBasicInput) trans() (params map[string]string, headers map[ return } -func (input PutObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.PutObjectBasicInput.trans() +func (input PutObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.PutObjectBasicInput.trans(isObs) + if err != nil { + return + } if input.Body != nil { data = input.Body } return } -func (input CopyObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ObjectOperationInput.trans() +func (input CopyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } var copySource string if input.CopySourceVersionId != "" { - copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, input.CopySourceKey, input.CopySourceVersionId) + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) } else { - copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, input.CopySourceKey) + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) } - headers[HEADER_COPY_SOURCE_AMZ] = []string{copySource} + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) if directive := string(input.MetadataDirective); directive != "" { - headers[HEADER_METADATA_DIRECTIVE_AMZ] = []string{directive} + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{directive}, isObs) } if input.MetadataDirective == ReplaceMetadata { @@ -463,55 +664,67 @@ func (input CopyObjectInput) trans() (params map[string]string, headers map[stri } if input.CopySourceIfMatch != "" { - headers[HEADER_COPY_SOURCE_IF_MATCH_AMZ] = []string{input.CopySourceIfMatch} + setHeaders(headers, HEADER_COPY_SOURCE_IF_MATCH, []string{input.CopySourceIfMatch}, isObs) } if input.CopySourceIfNoneMatch != "" { - headers[HEADER_COPY_SOURCE_IF_NONE_MATCH_AMZ] = []string{input.CopySourceIfNoneMatch} + setHeaders(headers, HEADER_COPY_SOURCE_IF_NONE_MATCH, []string{input.CopySourceIfNoneMatch}, isObs) } if !input.CopySourceIfModifiedSince.IsZero() { - headers[HEADER_COPY_SOURCE_IF_MODIFIED_SINCE_AMZ] = []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)} + setHeaders(headers, HEADER_COPY_SOURCE_IF_MODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}, isObs) } if !input.CopySourceIfUnmodifiedSince.IsZero() { - headers[HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE_AMZ] = []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)} + setHeaders(headers, HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}, isObs) } if input.SourceSseHeader != nil { if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { - headers[HEADER_SSEC_COPY_SOURCE_ENCRYPTION_AMZ] = []string{sseCHeader.GetEncryption()} - headers[HEADER_SSEC_COPY_SOURCE_KEY_AMZ] = []string{sseCHeader.GetKey()} - headers[HEADER_SSEC_COPY_SOURCE_KEY_MD5_AMZ] = []string{sseCHeader.GetKeyMD5()} + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) } } + if input.SuccessActionRedirect != "" { + headers[HEADER_SUCCESS_ACTION_REDIRECT] = []string{input.SuccessActionRedirect} + } return } -func (input AbortMultipartUploadInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input AbortMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId} return } -func (input InitiateMultipartUploadInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ObjectOperationInput.trans() +func (input InitiateMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } params[string(SubResourceUploads)] = "" return } -func (input UploadPartInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input UploadPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} headers = make(map[string][]string) - setSseHeader(headers, input.SseHeader, true) + setSseHeader(headers, input.SseHeader, true, isObs) + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } if input.Body != nil { data = input.Body } return } -func (input CompleteMultipartUploadInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input CompleteMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId} data, _ = ConvertCompleteMultipartUploadInputToXml(input, false) return } -func (input ListPartsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId} if input.MaxParts > 0 { params["max-parts"] = IntToString(input.MaxParts) @@ -522,28 +735,28 @@ func (input ListPartsInput) trans() (params map[string]string, headers map[strin return } -func (input CopyPartInput) trans() (params map[string]string, headers map[string][]string, data interface{}) { +func (input CopyPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} headers = make(map[string][]string, 1) var copySource string if input.CopySourceVersionId != "" { - copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, input.CopySourceKey, input.CopySourceVersionId) + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) } else { - copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, input.CopySourceKey) + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) } - headers[HEADER_COPY_SOURCE_AMZ] = []string{copySource} - + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) if input.CopySourceRangeStart >= 0 && input.CopySourceRangeEnd > input.CopySourceRangeStart { - headers[HEADER_COPY_SOURCE_RANGE_AMZ] = []string{fmt.Sprintf("bytes=%d-%d", input.CopySourceRangeStart, input.CopySourceRangeEnd)} + setHeaders(headers, HEADER_COPY_SOURCE_RANGE, []string{fmt.Sprintf("bytes=%d-%d", input.CopySourceRangeStart, input.CopySourceRangeEnd)}, isObs) } - setSseHeader(headers, input.SseHeader, true) + setSseHeader(headers, input.SseHeader, true, isObs) if input.SourceSseHeader != nil { if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { - headers[HEADER_SSEC_COPY_SOURCE_ENCRYPTION_AMZ] = []string{sseCHeader.GetEncryption()} - headers[HEADER_SSEC_COPY_SOURCE_KEY_AMZ] = []string{sseCHeader.GetKey()} - headers[HEADER_SSEC_COPY_SOURCE_KEY_MD5_AMZ] = []string{sseCHeader.GetKeyMD5()} + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) } + } return } diff --git a/openstack/obs/util.go b/openstack/obs/util.go index 7edadadd9..b685b81b7 100644 --- a/openstack/obs/util.go +++ b/openstack/obs/util.go @@ -1,3 +1,15 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + package obs import ( @@ -8,6 +20,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/xml" + "fmt" "net/url" "regexp" "strconv" @@ -16,7 +29,21 @@ import ( ) var regex = regexp.MustCompile("^[\u4e00-\u9fa5]$") +var ipRegex = regexp.MustCompile("^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$") +var v4AuthRegex = regexp.MustCompile("Credential=(.+?),SignedHeaders=(.+?),Signature=.+") +var regionRegex = regexp.MustCompile(".+/\\d+/(.+?)/.+") +func StringContains(src string, subStr string, subTranscoding string) string { + return strings.Replace(src, subStr, subTranscoding, -1) +} +func XmlTranscoding(src string) string { + srcTmp := StringContains(src, "&", "&") + srcTmp = StringContains(srcTmp, "<", "<") + srcTmp = StringContains(srcTmp, ">", ">") + srcTmp = StringContains(srcTmp, "'", "'") + srcTmp = StringContains(srcTmp, "\"", """) + return srcTmp +} func StringToInt(value string, def int) int { ret, err := strconv.Atoi(value) if err != nil { @@ -56,19 +83,28 @@ func FormatUtcToRfc1123(t time.Time) string { func Md5(value []byte) []byte { m := md5.New() - m.Write(value) + _, err := m.Write(value) + if err != nil { + doLog(LEVEL_WARN, "MD5 failed to write with reason: %v", err) + } return m.Sum(nil) } func HmacSha1(key, value []byte) []byte { mac := hmac.New(sha1.New, key) - mac.Write(value) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha1 failed to write with reason: %v", err) + } return mac.Sum(nil) } func HmacSha256(key, value []byte) []byte { mac := hmac.New(sha256.New, key) - mac.Write(value) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha256 failed to write with reason: %v", err) + } return mac.Sum(nil) } @@ -90,7 +126,10 @@ func Base64Md5(value []byte) string { func Sha256Hash(value []byte) []byte { hash := sha256.New() - hash.Write(value) + _, err := hash.Write(value) + if err != nil { + doLog(LEVEL_WARN, "Sha256Hash failed to write with reason: %v", err) + } return hash.Sum(nil) } @@ -124,6 +163,21 @@ func UrlDecode(value string) (string, error) { return "", err } +func UrlDecodeWithoutError(value string) string { + ret, err := UrlDecode(value) + if err == nil { + return ret + } + if isErrorLogEnabled() { + doLog(LEVEL_ERROR, "Url decode error: %v", err) + } + return "" +} + +func IsIP(value string) bool { + return ipRegex.MatchString(value) +} + func UrlEncode(value string, chineseOnly bool) string { if chineseOnly { values := make([]string, 0, len(value)) @@ -138,3 +192,303 @@ func UrlEncode(value string, chineseOnly bool) string { } return url.QueryEscape(value) } + +func copyHeaders(m map[string][]string) (ret map[string][]string) { + if m != nil { + ret = make(map[string][]string, len(m)) + for key, values := range m { + _values := make([]string, 0, len(values)) + for _, value := range values { + _values = append(_values, value) + } + ret[strings.ToLower(key)] = _values + } + } else { + ret = make(map[string][]string) + } + + return +} + +func parseHeaders(headers map[string][]string) (signature string, region string, signedHeaders string) { + signature = "v2" + if receviedAuthorization, ok := headers[strings.ToLower(HEADER_AUTH_CAMEL)]; ok && len(receviedAuthorization) > 0 { + if strings.HasPrefix(receviedAuthorization[0], V4_HASH_PREFIX) { + signature = "v4" + matches := v4AuthRegex.FindStringSubmatch(receviedAuthorization[0]) + if len(matches) >= 3 { + region = matches[1] + regions := regionRegex.FindStringSubmatch(region) + if len(regions) >= 2 { + region = regions[1] + } + signedHeaders = matches[2] + } + + } else if strings.HasPrefix(receviedAuthorization[0], V2_HASH_PREFIX) { + signature = "v2" + } + } + return +} + +func getTemporaryKeys() []string { + return []string{ + "Signature", + "signature", + "X-Amz-Signature", + "x-amz-signature", + } +} + +func getIsObs(isTemporary bool, querys []string, headers map[string][]string) bool { + isObs := true + if isTemporary { + for _, value := range querys { + keyPrefix := strings.ToLower(value) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + } else if strings.HasPrefix(value, HEADER_ACCESSS_KEY_AMZ) { + isObs = false + } + } + } else { + for key, _ := range headers { + keyPrefix := strings.ToLower(key) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + break + } + } + } + return isObs +} + +func GetV2Authorization(ak, sk, method, bucketName, objectKey, queryUrl string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryUrl, "?") { + queryUrl = queryUrl[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryUrl, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + headers = copyHeaders(headers) + pathStyle := false + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + pathStyle = true + } + conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + conf.signature = SignatureObs + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedURL, headers, true) + v2HashPrefix := OBS_HASH_PREFIX + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + return +} + +func GetAuthorization(ak, sk, method, bucketName, objectKey, queryUrl string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryUrl, "?") { + queryUrl = queryUrl[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryUrl, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + isTemporary := false + signature := "v2" + temporaryKeys := getTemporaryKeys() + for _, key := range temporaryKeys { + if _, ok := params[key]; ok { + isTemporary = true + if strings.ToLower(key) == "signature" { + signature = "v2" + } else if strings.ToLower(key) == "x-amz-signature" { + signature = "v4" + } + break + } + } + isObs := getIsObs(isTemporary, querysResult, headers) + headers = copyHeaders(headers) + pathStyle := false + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + pathStyle = true + } + conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + + if isTemporary { + return getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature, conf, params, headers, isObs) + } else { + signature, region, signedHeaders := parseHeaders(headers) + if signature == "v4" { + conf.signature = SignatureV4 + requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestUrl, _err := url.Parse(requestUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl with reason: %v", _err) + return nil + } + headerKeys := strings.Split(signedHeaders, ";") + _headers := make(map[string][]string, len(headerKeys)) + for _, headerKey := range headerKeys { + _headers[headerKey] = headers[headerKey] + } + ret = v4Auth(ak, sk, region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, _headers) + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedUrl, headers, isObs) + v2HashPrefix := V2_HASH_PREFIX + if isObs { + v2HashPrefix = OBS_HASH_PREFIX + } + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + } + return + } + +} + +func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature string, conf *config, params map[string]string, + headers map[string][]string, isObs bool) (ret map[string]string) { + + if signature == "v4" { + conf.signature = SignatureV4 + + longDate, ok := params[PARAM_DATE_AMZ_CAMEL] + if !ok { + longDate = params[HEADER_DATE_AMZ] + } + shortDate := longDate[:8] + + credential, ok := params[PARAM_CREDENTIAL_AMZ_CAMEL] + if !ok { + credential = params[strings.ToLower(PARAM_CREDENTIAL_AMZ_CAMEL)] + } + + _credential := UrlDecodeWithoutError(credential) + + regions := regionRegex.FindStringSubmatch(_credential) + var region string + if len(regions) >= 2 { + region = regions[1] + } + + _, scope := getCredential(ak, region, shortDate) + + expires, ok := params[PARAM_EXPIRES_AMZ_CAMEL] + if !ok { + expires = params[strings.ToLower(PARAM_EXPIRES_AMZ_CAMEL)] + } + + signedHeaders, ok := params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] + if !ok { + signedHeaders = params[strings.ToLower(PARAM_SIGNEDHEADERS_AMZ_CAMEL)] + } + + algorithm, ok := params[PARAM_ALGORITHM_AMZ_CAMEL] + if !ok { + algorithm = params[strings.ToLower(PARAM_ALGORITHM_AMZ_CAMEL)] + } + + if _, ok := params[PARAM_SIGNATURE_AMZ_CAMEL]; ok { + delete(params, PARAM_SIGNATURE_AMZ_CAMEL) + } else if _, ok := params[strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)]; ok { + delete(params, strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)) + } + + ret = make(map[string]string, 6) + ret[PARAM_ALGORITHM_AMZ_CAMEL] = algorithm + ret[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + ret[PARAM_DATE_AMZ_CAMEL] = longDate + ret[PARAM_EXPIRES_AMZ_CAMEL] = expires + ret[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = signedHeaders + + requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestUrl, _err := url.Parse(requestUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl with reason: %v", _err) + return nil + } + stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, strings.Split(signedHeaders, ";"), headers) + ret[PARAM_SIGNATURE_AMZ_CAMEL] = UrlEncode(getSignature(stringToSign, sk, region, shortDate), false) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + expires, ok := params["Expires"] + if !ok { + expires = params["expires"] + } + headers[HEADER_DATE_CAMEL] = []string{expires} + stringToSign := getV2StringToSign(method, canonicalizedUrl, headers, isObs) + ret = make(map[string]string, 3) + ret["Signature"] = UrlEncode(Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign))), false) + ret["AWSAccessKeyId"] = UrlEncode(ak, false) + ret["Expires"] = UrlEncode(expires, false) + } + + return +}