fix: choose candidate list from the list returned from backend

cmd/googlecli/main.go

@@ -18,17 +18,21 @@ import (
 	"fmt"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/multicloud/google"
 	_ "yunion.io/x/onecloud/pkg/multicloud/google/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 type BaseOptions struct {
 	Debug        bool   `help:"debug mode"`
 	Help         bool   `help:"Show help"`
+	AuthFile     string `help:"google cloud auth json file path" default:"$GOOGLE_AUTH_FILE"`
 	ClientEmail  string `help:"Client email" default:"$GOOGLE_CLIENT_EMAIL"`
 	ProjectID    string `help:"Project ID" default:"$GOOGLE_PROJECT_ID"`
 	PrivateKeyID string `help:"Private Key ID" default:"$GOOGLE_PRIVATE_KEY_ID"`
@@ -78,6 +82,20 @@ func showErrorAndExit(e error) {
 }
 
 func newClient(options *BaseOptions) (*google.SRegion, error) {
+	if len(options.AuthFile) > 0 {
+		jsonStr, err := fileutils2.FileGetContents(options.AuthFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "FileGetContents")
+		}
+		jsonCfg, err := jsonutils.ParseString(jsonStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "jsonutils.ParseString")
+		}
+		options.ClientEmail, _ = jsonCfg.GetString("client_email")
+		options.PrivateKeyID, _ = jsonCfg.GetString("private_key_id")
+		options.PrivateKey, _ = jsonCfg.GetString("private_key")
+		options.ProjectID, _ = jsonCfg.GetString("project_id")
+	}
 	if len(options.ClientEmail) == 0 {
 		return nil, fmt.Errorf("Missing ClientEmail")
 	}

pkg/apis/compute/cloudprovider.go

@@ -251,3 +251,23 @@ type CloudproviderUpdateInput struct {
 
 type CloudproviderCreateInput struct {
 }
+
+type CloudproviderGetStorageClassInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetStorageClassOutput struct {
+	// 对象存储存储类型
+	StorageClasses []string `json:"storage_classes"`
+}
+
+type CloudproviderGetCannedAclInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetCannedAclOutput struct {
+	// Bucket支持的预置ACL列表
+	BucketCannedAcls []string `json:"bucket_canned_acls"`
+	// Object支持的预置ACL列表
+	ObjectCannedAcls []string `json:"object_canned_acls"`
+}

pkg/cloudprovider/cloudprovider.go

@@ -224,6 +224,8 @@ type ICloudProvider interface {
 	GetCloudRegionExternalIdPrefix() string
 
 	GetStorageClasses(regionId string) []string
+	GetBucketCannedAcls(regionId string) []string
+	GetObjectCannedAcls(regionId string) []string
 
 	GetCapabilities() []string
 	GetICloudQuotas() ([]ICloudQuota, error)

pkg/compute/models/buckets.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -420,6 +421,16 @@ func (manager *SBucketManager) ValidateCreateData(
 		return input, httperrors.NewInputParameterError("invalid bucket name %s: %s", input.Name, err)
 	}
 
+	if len(input.StorageClass) > 0 {
+		driver, err := managerV.GetProvider()
+		if err != nil {
+			return input, errors.Wrap(err, "GetProvider")
+		}
+		if !utils.IsInStringArray(input.StorageClass, driver.GetStorageClasses(cloudRegionV.Id)) {
+			return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid storage class %s", input.StorageClass)
+		}
+	}
+
 	quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, cloudRegionV, managerV)
 	pendingUsage := SRegionQuota{Bucket: 1}
 	pendingUsage.SetKeys(quotaKeys)
@@ -975,14 +986,17 @@ func (bucket *SBucket) PerformUpload(
 		return nil, httperrors.NewInputParameterError("Content-Length negative %d", sizeBytes)
 	}
 	storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER)
+	driver, err := bucket.GetDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetDriver")
+	}
+	if len(storageClass) > 0 && !utils.IsInStringArray(storageClass, driver.GetStorageClasses(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "invalid storage class %s", storageClass)
+	}
+
 	aclStr := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER)
-	if len(aclStr) > 0 {
-		switch cloudprovider.TBucketACLType(aclStr) {
-		case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
-			// do nothing
-		default:
-			return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr)
-		}
+	if len(aclStr) > 0 && !utils.IsInStringArray(aclStr, driver.GetObjectCannedAcls(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "invalid acl %s", aclStr)
 	}
 
 	inc := cloudprovider.SBucketStats{}
@@ -1060,15 +1074,23 @@ func (bucket *SBucket) PerformAcl(
 ) (jsonutils.JSONObject, error) {
 	err := input.Validate()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "ValidateInput")
+	}
+
+	provider, err := bucket.GetDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetDriver")
 	}
 
 	iBucket, objects, err := bucket.processObjectsActionInput(input.BucketObjectsActionInput)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "processObjectsActionInput")
 	}
 
 	if len(objects) == 0 {
+		if !utils.IsInStringArray(string(input.Acl), provider.GetBucketCannedAcls(bucket.CloudregionId)) {
+			return nil, errors.Wrapf(httperrors.ErrInputParameter, "unsupported bucket canned acl %s", input.Acl)
+		}
 		err = iBucket.SetAcl(input.Acl)
 		if err != nil {
 			return nil, httperrors.NewInternalServerError("setAcl error %s", err)
@@ -1081,6 +1103,10 @@ func (bucket *SBucket) PerformAcl(
 		return nil, nil
 	}
 
+	if !utils.IsInStringArray(string(input.Acl), provider.GetObjectCannedAcls(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "unsupported object canned acl %s", input.Acl)
+	}
+
 	errs := make([]error, 0)
 	for _, object := range objects {
 		err := object.SetAcl(input.Acl)

pkg/compute/models/cloudproviders.go

@@ -1560,33 +1560,56 @@ func (provider *SCloudprovider) AllowGetDetailsStorageClasses(
 func (provider *SCloudprovider) GetDetailsStorageClasses(
 	ctx context.Context,
 	userCred mcclient.TokenCredential,
-	query jsonutils.JSONObject,
-) (jsonutils.JSONObject, error) {
+	input api.CloudproviderGetStorageClassInput,
+) (api.CloudproviderGetStorageClassOutput, error) {
+	output := api.CloudproviderGetStorageClassOutput{}
 	driver, err := provider.GetProvider()
 	if err != nil {
-		return nil, httperrors.NewInternalServerError("fail to get provider driver %s", err)
+		return output, httperrors.NewInternalServerError("fail to get provider driver %s", err)
 	}
-	extId := ""
-	regionStr := jsonutils.GetAnyString(query, []string{"cloudregion", "cloudregion_id"})
-	if len(regionStr) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
+	if len(input.Cloudregion) > 0 {
+		_, input.CloudregionResourceInput, err = ValidateCloudregionResourceInput(userCred, input.CloudregionResourceInput)
 		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
+			return output, errors.Wrap(err, "ValidateCloudregionResourceInput")
 		}
-		extId = regionObj.(*SCloudregion).GetExternalId()
 	}
 
-	sc := driver.GetStorageClasses(extId)
+	sc := driver.GetStorageClasses(input.Cloudregion)
 	if sc == nil {
-		return nil, httperrors.NewInternalServerError("storage classes not supported")
+		return output, httperrors.NewInternalServerError("storage classes not supported")
+	}
+	output.StorageClasses = sc
+	return output, nil
+}
+
+func (provider *SCloudprovider) AllowGetDetailsCannedAcls(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) bool {
+	return db.IsAdminAllowGetSpec(userCred, provider, "canned-acls")
+}
+
+func (provider *SCloudprovider) GetDetailsCannedAcls(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	input api.CloudproviderGetCannedAclInput,
+) (api.CloudproviderGetCannedAclOutput, error) {
+	output := api.CloudproviderGetCannedAclOutput{}
+	driver, err := provider.GetProvider()
+	if err != nil {
+		return output, httperrors.NewInternalServerError("fail to get provider driver %s", err)
 	}
-	ret := jsonutils.NewDict()
-	ret.Add(jsonutils.NewStringArray(sc), "storage_classes")
-	return ret, nil
+	if len(input.Cloudregion) > 0 {
+		_, input.CloudregionResourceInput, err = ValidateCloudregionResourceInput(userCred, input.CloudregionResourceInput)
+		if err != nil {
+			return output, errors.Wrap(err, "ValidateCloudregionResourceInput")
+		}
+	}
+
+	output.BucketCannedAcls = driver.GetBucketCannedAcls(input.Cloudregion)
+	output.ObjectCannedAcls = driver.GetObjectCannedAcls(input.Cloudregion)
+	return output, nil
 }
 
 func (provider *SCloudprovider) getAccountShareInfo() apis.SAccountShareInfo {

pkg/multicloud/aliyun/provider/provider.go

@@ -169,6 +169,22 @@ func (self *SAliyunProvider) GetStorageClasses(regionId string) []string {
 	}
 }
 
+func (self *SAliyunProvider) GetBucketCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLPublicRead),
+		string(cloudprovider.ACLPublicReadWrite),
+	}
+}
+
+func (self *SAliyunProvider) GetObjectCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLPublicRead),
+		string(cloudprovider.ACLPublicReadWrite),
+	}
+}
+
 func (self *SAliyunProvider) GetCapabilities() []string {
 	return self.client.GetCapabilities()
 }

pkg/multicloud/aws/aws.go

@@ -514,3 +514,35 @@ func (client *SAwsClient) GetIamLoginUrl() string {
 		return fmt.Sprintf("https://%s.signin.aws.amazon.com/console/", identity.Account)
 	}
 }
+
+func (client *SAwsClient) GetBucketCannedAcls() []string {
+	switch client.GetAccessEnv() {
+	case api.CLOUD_ACCESS_ENV_AWS_CHINA:
+		return []string{
+			string(cloudprovider.ACLPrivate),
+		}
+	default:
+		return []string{
+			string(cloudprovider.ACLPrivate),
+			string(cloudprovider.ACLAuthRead),
+			string(cloudprovider.ACLPublicRead),
+			string(cloudprovider.ACLPublicReadWrite),
+		}
+	}
+}
+
+func (client *SAwsClient) GetObjectCannedAcls() []string {
+	switch client.GetAccessEnv() {
+	case api.CLOUD_ACCESS_ENV_AWS_CHINA:
+		return []string{
+			string(cloudprovider.ACLPrivate),
+		}
+	default:
+		return []string{
+			string(cloudprovider.ACLPrivate),
+			string(cloudprovider.ACLAuthRead),
+			string(cloudprovider.ACLPublicRead),
+			string(cloudprovider.ACLPublicReadWrite),
+		}
+	}
+}

pkg/multicloud/aws/provider/provider.go

@@ -168,6 +168,14 @@ func (self *SAwsProvider) GetStorageClasses(regionId string) []string {
 	}
 }
 
+func (self *SAwsProvider) GetBucketCannedAcls(regionId string) []string {
+	return self.client.GetBucketCannedAcls()
+}
+
+func (self *SAwsProvider) GetObjectCannedAcls(regionId string) []string {
+	return self.client.GetObjectCannedAcls()
+}
+
 func (self *SAwsProvider) GetCloudRegionExternalIdPrefix() string {
 	return self.client.GetAccessEnv() + "/"
 }

pkg/multicloud/azure/provider/provider.go

@@ -210,6 +210,20 @@ func (self *SAzureProvider) GetStorageClasses(regionId string) []string {
 	return sc
 }
 
+func (self *SAzureProvider) GetBucketCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLPublicRead),
+	}
+}
+
+func (self *SAzureProvider) GetObjectCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLPublicRead),
+	}
+}
+
 func (self *SAzureProvider) GetCloudRegionExternalIdPrefix() string {
 	return self.client.GetAccessEnv() + "/"
 }

pkg/multicloud/ctyun/provider/provider.go

@@ -160,6 +160,14 @@ func (self *SCtyunProvider) GetStorageClasses(regionId string) []string {
 	}
 }
 
+func (self *SCtyunProvider) GetBucketCannedAcls(regionId string) []string {
+	return nil
+}
+
+func (self *SCtyunProvider) GetObjectCannedAcls(regionId string) []string {
+	return nil
+}
+
 func (self *SCtyunProvider) GetCloudRegionExternalIdPrefix() string {
 	return self.client.GetCloudRegionExternalIdPrefix()
 }

pkg/multicloud/esxi/provider/provider.go

@@ -193,6 +193,14 @@ func (self *SESXiProvider) GetStorageClasses(regionId string) []string {
 	return nil
 }
 
+func (self *SESXiProvider) GetBucketCannedAcls(regionId string) []string {
+	return nil
+}
+
+func (self *SESXiProvider) GetObjectCannedAcls(regionId string) []string {
+	return nil
+}
+
 func (self *SESXiProvider) GetCapabilities() []string {
 	return self.client.GetCapabilities()
 }

pkg/multicloud/google/provider/provider.go

@@ -232,6 +232,24 @@ func (self *SGoogleProvider) GetStorageClasses(regionId string) []string {
 	}
 }
 
+func (self *SGoogleProvider) GetBucketCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLAuthRead),
+		string(cloudprovider.ACLPublicRead),
+		string(cloudprovider.ACLPublicReadWrite),
+	}
+}
+
+func (self *SGoogleProvider) GetObjectCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLAuthRead),
+		string(cloudprovider.ACLPublicRead),
+		string(cloudprovider.ACLPublicReadWrite),
+	}
+}
+
 func (self *SGoogleProvider) GetCapabilities() []string {
 	return self.client.GetCapabilities()
 }