fix: 避免未本地缓存usercache

pkg/cloudid/models/cloudaccount.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"net/http"
 	"net/url"
+	"time"
 
 	"golang.org/x/net/http/httpproxy"
 
@@ -48,6 +49,7 @@ type SCloudaccountManager struct {
 }
 
 var CloudaccountManager *SCloudaccountManager
+var isCloudacountSynced bool
 
 func init() {
 	CloudaccountManager = &SCloudaccountManager{
@@ -59,6 +61,7 @@ func init() {
 		),
 	}
 	CloudaccountManager.SetVirtualObject(CloudaccountManager)
+	isCloudacountSynced = false
 }
 
 type SCloudaccount struct {
@@ -279,6 +282,15 @@ func (manager *SCloudaccountManager) SyncCloudaccounts(ctx context.Context, user
 		result = account.syncCloudprovider(ctx, userCred)
 		log.Infof("sync cloudprovider for cloudaccount %s(%s) result: %s", account.Name, account.Id, result.Result())
 	}
+	isCloudacountSynced = true
+}
+
+// 避免第一次启动时，云账号列表为空，子账号及其他资源需要等待一个周期才能同步
+func waitForSync(task string) {
+	for isCloudacountSynced == false {
+		log.Debugf("cloudaccount not sync try later do task %s", task)
+		time.Sleep(time.Second * 30)
+	}
 }
 
 func (self SCloudaccount) GetGlobalId() string {
@@ -414,6 +426,7 @@ func (account *SCloudDelegate) GetProvider() (cloudprovider.ICloudProvider, erro
 }
 
 func (manager *SCloudaccountManager) SyncCloudusers(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	waitForSync("SyncCloudusersTask")
 	accounts, err := manager.GetCloudaccounts()
 	if err != nil {
 		log.Errorf("GetLocalCloudaccounts: %v", err)
@@ -594,7 +607,7 @@ func (self *SCloudaccount) SyncCloudpolicies(ctx context.Context, userCred mccli
 	}
 
 	for i := 0; i < len(added); i++ {
-		err := CloudpolicyManager.newFromCloudpolicy(ctx, userCred, added[i], self.Provider)
+		_, err := CloudpolicyManager.newFromCloudpolicy(ctx, userCred, added[i], self.Provider)
 		if err != nil {
 			result.AddError(err)
 			continue
@@ -716,6 +729,7 @@ func (manager *SCloudaccountManager) GetSupportCreateCloudgroupAccounts() ([]SCl
 }
 
 func (manager *SCloudaccountManager) SyncCloudpolicies(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	waitForSync("SyncCloudpoliciesTask")
 	accounts, err := manager.GetCloudaccounts()
 	if err != nil {
 		log.Errorf("GetCloudaccounts error: %v", err)
@@ -740,6 +754,7 @@ func (self *SCloudaccount) StartSyncCloudpolicyTask(ctx context.Context, userCre
 }
 
 func (manager *SCloudaccountManager) SyncCloudgroups(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	waitForSync("SyncCloudgroupsTask")
 	accounts, err := manager.GetSupportCreateCloudgroupAccounts()
 	if err != nil {
 		log.Errorf("GetSupportCreateCloudgroupAccounts error: %v", err)
@@ -804,12 +819,7 @@ func (self *SCloudaccount) SyncCloudgroupcaches(ctx context.Context, userCred mc
 	}
 
 	for i := 0; i < len(added); i++ {
-		group, err := self.GetOrCreateCloudgroup(ctx, userCred, added[i])
-		if err != nil {
-			result.AddError(err)
-			continue
-		}
-		err = CloudgroupcacheManager.newFromCloudgroup(ctx, userCred, added[i], group, self.Id)
+		_, err := self.newCloudgroup(ctx, userCred, added[i])
 		if err != nil {
 			result.AddError(err)
 			continue
@@ -819,6 +829,18 @@ func (self *SCloudaccount) SyncCloudgroupcaches(ctx context.Context, userCred mc
 	return result
 }
 
+func (self *SCloudaccount) newCloudgroup(ctx context.Context, userCred mcclient.TokenCredential, iGroup cloudprovider.ICloudgroup) (*SCloudgroupcache, error) {
+	group, err := self.GetOrCreateCloudgroup(ctx, userCred, iGroup)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetOrCreateCloudgroup")
+	}
+	cache, err := CloudgroupcacheManager.newFromCloudgroup(ctx, userCred, iGroup, group, self.Id)
+	if err != nil {
+		return nil, errors.Wrap(err, "newFromCloudgroup")
+	}
+	return cache, nil
+}
+
 func (self *SCloudaccount) GetOrCreateCloudgroup(ctx context.Context, userCred mcclient.TokenCredential, iGroup cloudprovider.ICloudgroup) (*SCloudgroup, error) {
 	groups, err := self.GetCloudgroups()
 	if err != nil {
@@ -828,6 +850,20 @@ func (self *SCloudaccount) GetOrCreateCloudgroup(ctx context.Context, userCred m
 	if err != nil {
 		return nil, errors.Wrap(err, "GetICloudpolicies")
 	}
+
+	for i := range iPolicies {
+		_, err := db.FetchByExternalId(CloudpolicyManager, iPolicies[i].GetGlobalId())
+		if err == nil {
+			continue
+		}
+		if errors.Cause(err) != sql.ErrNoRows {
+			return nil, errors.Wrapf(err, "db.FetchByExternalId(%s)", iPolicies[i].GetGlobalId())
+		}
+		_, err = CloudpolicyManager.newFromCloudpolicy(ctx, userCred, iPolicies[i], self.Provider)
+		if err != nil {
+			return nil, errors.Wrap(err, "newFromCloudpolicy")
+		}
+	}
 	for i := range groups {
 		isEqual, err := groups[i].IsEqual(iPolicies)
 		if err != nil {

pkg/cloudid/models/cloudgroup.go

@@ -756,7 +756,7 @@ func (self *SCloudgroup) IsEqual(iPolicies []cloudprovider.ICloudpolicy) (bool,
 	if err != nil {
 		return false, errors.Wrap(err, "CompareSets")
 	}
-	return len(iPolicies) == len(commondb), nil
+	return len(removed)+len(added) == 0, nil
 }
 
 func (self *SCloudgroup) attachPolicyFromCloudpolicy(ctx context.Context, userCred mcclient.TokenCredential, iPolicy cloudprovider.ICloudpolicy) error {
@@ -767,6 +767,13 @@ func (self *SCloudgroup) attachPolicyFromCloudpolicy(ctx context.Context, userCr
 	if err != nil {
 		return errors.Wrapf(err, "db.FetchByExternalId(%s)", iPolicy.GetGlobalId())
 	}
-	up.CloudpolicyId = p.GetId()
-	return CloudgroupPolicyManager.TableSpec().Insert(ctx, up)
+	_, err = self.GetCloudpolicy(p.GetId())
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			up.CloudpolicyId = p.GetId()
+			return CloudgroupPolicyManager.TableSpec().Insert(ctx, up)
+		}
+		return errors.Wrapf(err, "GetCloudpolicy(%s)", p.GetId())
+	}
+	return nil
 }

pkg/cloudid/models/cloudgroupcaches.go

@@ -119,7 +119,7 @@ func (self *SCloudgroupcache) RealDelete(ctx context.Context, userCred mcclient.
 	return self.SStatusStandaloneResourceBase.Delete(ctx, userCred)
 }
 
-func (manager *SCloudgroupcacheManager) newFromCloudgroup(ctx context.Context, userCred mcclient.TokenCredential, iGroup cloudprovider.ICloudgroup, group *SCloudgroup, cloudaccountId string) error {
+func (manager *SCloudgroupcacheManager) newFromCloudgroup(ctx context.Context, userCred mcclient.TokenCredential, iGroup cloudprovider.ICloudgroup, group *SCloudgroup, cloudaccountId string) (*SCloudgroupcache, error) {
 	cache := &SCloudgroupcache{}
 	cache.SetModelManager(manager, cache)
 	cache.CloudgroupId = group.Id
@@ -128,7 +128,7 @@ func (manager *SCloudgroupcacheManager) newFromCloudgroup(ctx context.Context, u
 	cache.Status = api.CLOUD_GROUP_STATUS_AVAILABLE
 	cache.ExternalId = iGroup.GetGlobalId()
 	cache.CloudaccountId = cloudaccountId
-	return manager.TableSpec().Insert(ctx, cache)
+	return cache, manager.TableSpec().Insert(ctx, cache)
 }
 
 func (self *SCloudgroupcache) syncWithCloudgrup(ctx context.Context, userCred mcclient.TokenCredential, iGroup cloudprovider.ICloudgroup) error {

pkg/cloudid/models/cloudpolicy.go

@@ -254,7 +254,7 @@ func (self *SCloudpolicy) AllowDeleteItem(ctx context.Context, userCred mcclient
 	return false
 }
 
-func (manager *SCloudpolicyManager) newFromCloudpolicy(ctx context.Context, userCred mcclient.TokenCredential, iPolicy cloudprovider.ICloudpolicy, provider string) error {
+func (manager *SCloudpolicyManager) newFromCloudpolicy(ctx context.Context, userCred mcclient.TokenCredential, iPolicy cloudprovider.ICloudpolicy, provider string) (*SCloudpolicy, error) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -266,7 +266,7 @@ func (manager *SCloudpolicyManager) newFromCloudpolicy(ctx context.Context, user
 	policy.Provider = provider
 	policy.ExternalId = iPolicy.GetGlobalId()
 	policy.Description = iPolicy.GetDescription()
-	return manager.TableSpec().Insert(ctx, policy)
+	return policy, manager.TableSpec().Insert(ctx, policy)
 }
 
 func (self *SCloudpolicy) SyncWithCloudpolicy(ctx context.Context, userCred mcclient.TokenCredential, iPolicy cloudprovider.ICloudpolicy) error {

pkg/cloudid/models/clouduser.go

@@ -276,6 +276,10 @@ func (manager *SClouduserManager) ValidateCreateData(ctx context.Context, userCr
 	if err != nil {
 		return input, httperrors.NewGeneralError(errors.Wrap(err, "FetchAccount"))
 	}
+	// 只有系统管理员和账号所在的域管理员可以创建子用户
+	if !((account.DomainId == userCred.GetProjectDomainId() && db.IsDomainAllowCreate(userCred, manager)) || userCred.HasSystemAdminPrivilege()) {
+		return input, httperrors.NewForbiddenError("forbidden to create clouduser for cloudaccount %s", account.Name)
+	}
 	if !account.IsSupportCloudId.Bool() {
 		return input, httperrors.NewUnsupportOperationError("account %s not support create clouduser", account.Name)
 	}
@@ -564,13 +568,29 @@ func (self *SClouduser) SyncCloudgroups(ctx context.Context, userCred mcclient.T
 	result.UpdateCnt = len(commondb)
 
 	for i := 0; i < len(added); i++ {
+		var cloudgroupId string
 		_cache, err := db.FetchByExternalId(CloudgroupcacheManager, added[i].GetGlobalId())
 		if err != nil {
-			result.AddError(errors.Wrapf(err, "FetchByExternalId(%s)", added[i].GetGlobalId()))
-			continue
+			if errors.Cause(err) != sql.ErrNoRows {
+				result.AddError(errors.Wrapf(err, "FetchByExternalId(%s)", added[i].GetGlobalId()))
+				continue
+			}
+			account, err := self.GetCloudaccount()
+			if err != nil {
+				result.AddError(errors.Wrap(err, "GetCloudaccount"))
+				continue
+			}
+			cache, err := account.newCloudgroup(ctx, userCred, added[i])
+			if err != nil {
+				result.AddError(errors.Wrap(err, "account.newCloudgroup"))
+				continue
+			}
+			cloudgroupId = cache.CloudgroupId
+		} else {
+			cache := _cache.(*SCloudgroupcache)
+			cloudgroupId = cache.CloudgroupId
 		}
-		cache := _cache.(*SCloudgroupcache)
-		err = self.joinGroup(cache.CloudgroupId)
+		err = self.joinGroup(cloudgroupId)
 		if err != nil {
 			result.AddError(errors.Wrap(err, "joinGroup"))
 			continue
@@ -1159,7 +1179,7 @@ func (self *SClouduser) AllowPerformChangeOwner(ctx context.Context, userCred mc
 
 // 变更子账号所属本地用户
 func (self *SClouduser) PerformChangeOwner(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ClouduserChangeOwnerInput) (jsonutils.JSONObject, error) {
-	user, err := db.UserCacheManager.FetchById(input.UserId)
+	user, err := db.UserCacheManager.FetchUserById(ctx, input.UserId)
 	if err != nil {
 		return nil, httperrors.NewGeneralError(errors.Wrapf(err, "Not found user %s", input.UserId))
 	}

pkg/cloudid/models/clouduser_policies.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -199,9 +200,21 @@ func (manager *SClouduserPolicyManager) newFromClouduserPolicy(ctx context.Conte
 
 	p, err := db.FetchByExternalId(CloudpolicyManager, iPolicy.GetGlobalId())
 	if err != nil {
-		return errors.Wrapf(err, "db.FetchByExternalId(%s)", iPolicy.GetGlobalId())
+		if errors.Cause(err) != sql.ErrNoRows {
+			return errors.Wrapf(err, "db.FetchByExternalId(%s)", iPolicy.GetGlobalId())
+		}
+		account, err := user.GetCloudaccount()
+		if err != nil {
+			return errors.Wrap(err, "user.GetCloudaccount")
+		}
+		policy, err := CloudpolicyManager.newFromCloudpolicy(ctx, userCred, iPolicy, account.Provider)
+		if err != nil {
+			return errors.Wrap(err, "newFromCloudpolicy")
+		}
+		up.CloudpolicyId = policy.Id
+	} else {
+		up.CloudpolicyId = p.GetId()
 	}
-	up.CloudpolicyId = p.GetId()
 
 	return manager.TableSpec().Insert(ctx, up)
 }